Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b58e752952aed4677522ad8389334c4861f2539aa80e417fc458133fc3e9c5ab
+oid sha256:d9a6a7c6cc358f69e338a4fc72fe10ecace28fbc9407592dc77dcfbaa63e9bc9
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d7ecb2ac19189bcf2c233b0cead6a8a4acbb6f35226ebd36c13702b9b92fd23
+oid sha256:1304f7ee646ec438fa036e322fe5faba64278f1ced0dc4b6609282dec05c0c04
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51be3507e4858df120a8f607c701c436099c60ecffd12f4bdd16f4abede4cc55
+oid sha256:1993809512de11da5c9bfc7edcfd772a774dbf6f9337aad97bbed0da7e7a2c50
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f51f9073d820deca33496502793f5bfc1491af5bcb6df0fcc33f6c729a1edcd
+oid sha256:29d214731ac218097d300baf3a5fec1ad7f997bc84f11b3da60d671a8faf4a3a
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5534246575342465,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=7-learning_rate=7.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.920/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.3915156871409633,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=8-learning_rate=2.0e-05-warmup_ratio=0.020-min_lr=0.000-factor=0.820/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.5123828649520874,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.8374,
+      "grad_norm": 1.0092626810073853,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 1.3581,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.3902701139450073,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.1232,
+      "grad_norm": 0.6695255637168884,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.3853,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.24360212683677673,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0412,
+      "grad_norm": 0.5797414183616638,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.2014,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.1984861195087433,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0223,
+      "grad_norm": 0.49628201127052307,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.1258,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.15782813727855682,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0142,
+      "grad_norm": 0.3971586525440216,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.0839,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.13507866859436035,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0106,
+      "grad_norm": 0.30195024609565735,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.0596,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.1490422487258911,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0084,
+      "grad_norm": 0.2556217610836029,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.0446,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.13086368143558502,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0088,
+      "grad_norm": 0.2777057886123657,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.0349,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.10114933550357819,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.006,
+      "grad_norm": 0.246878519654274,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.028,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.11398591101169586,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0052,
+      "grad_norm": 0.27177679538726807,
+      "learning_rate": 1.9500595004156286e-05,
+      "loss": 0.0231,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.5277065841802916,
-      "eval_e3_equal": 0.8086610693769333,
+      "eval_all_ligands_equal": 0.3915156871409633,
+      "eval_e3_equal": 0.7765797613787009,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9885992045956694,
-      "eval_e3_heavy_atoms_difference": 0.5718073353954927,
-      "eval_e3_heavy_atoms_difference_norm": 0.011028269140755255,
+      "eval_e3_has_attachment_point(s)": 0.9932832523199293,
+      "eval_e3_heavy_atoms_difference": 0.2896155545735749,
+      "eval_e3_heavy_atoms_difference_norm": 0.0020662380654388832,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9885992045956694,
-      "eval_has_all_attachment_points": 0.98285461776403,
-      "eval_has_three_substructures": 0.9992929739284137,
-      "eval_heavy_atoms_difference": 6.29509500662837,
-      "eval_heavy_atoms_difference_norm": 0.08120911067090385,
-      "eval_linker_equal": 0.7555457357490057,
+      "eval_e3_valid": 0.9932832523199293,
+      "eval_has_all_attachment_points": 0.9840919133893062,
+      "eval_has_three_substructures": 0.9990278391515687,
+      "eval_heavy_atoms_difference": 8.075386654882898,
+      "eval_heavy_atoms_difference_norm": 0.10478609381959636,
+      "eval_linker_equal": 0.563323022536456,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9959346000883783,
-      "eval_linker_heavy_atoms_difference": 0.3596995139195758,
-      "eval_linker_heavy_atoms_difference_norm": -0.0012085891768363632,
+      "eval_linker_has_attachment_point(s)": 0.9960229783473266,
+      "eval_linker_heavy_atoms_difference": 0.9376049491825011,
+      "eval_linker_heavy_atoms_difference_norm": 0.02394921771244689,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9959346000883783,
-      "eval_loss": 0.301065057516098,
-      "eval_num_fragments": 2.999646486964207,
-      "eval_poi_equal": 0.7543084401237296,
+      "eval_linker_valid": 0.9960229783473266,
+      "eval_loss": 0.286180704832077,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.7011047282368538,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9399911621741052,
-      "eval_poi_heavy_atoms_difference": 1.8794520547945206,
-      "eval_poi_heavy_atoms_difference_norm": 0.05591765226265377,
+      "eval_poi_has_attachment_point(s)": 0.9188687582854618,
+      "eval_poi_heavy_atoms_difference": 2.6504639858594787,
+      "eval_poi_heavy_atoms_difference_norm": 0.0799410695670892,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9399911621741052,
-      "eval_reassembly": 0.5353071144498454,
-      "eval_reassembly_nostereo": 0.5547503314184711,
-      "eval_runtime": 1499.4399,
-      "eval_samples_per_second": 7.546,
-      "eval_steps_per_second": 0.118,
+      "eval_poi_valid": 0.9188687582854618,
+      "eval_reassembly": 0.39752540874944764,
+      "eval_reassembly_nostereo": 0.41688024745912505,
+      "eval_runtime": 1468.607,
+      "eval_samples_per_second": 7.705,
+      "eval_steps_per_second": 0.121,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9274414494034467,
+      "eval_valid": 0.9114449845338047,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.10655809193849564,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0054,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.10990337282419205,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0048,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.0940764844417572,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0064,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.10571976006031036,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0034,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.08957740664482117,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.003,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5560760053026955,
-      "eval_e3_equal": 0.8172337604949182,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.989924878479894,
-      "eval_e3_heavy_atoms_difference": 0.5365444100751215,
-      "eval_e3_heavy_atoms_difference_norm": 0.011299086189019656,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.989924878479894,
-      "eval_has_all_attachment_points": 0.9816173221387539,
-      "eval_has_three_substructures": 0.9993813521873619,
-      "eval_heavy_atoms_difference": 6.706407423773752,
-      "eval_heavy_atoms_difference_norm": 0.08881506981006522,
-      "eval_linker_equal": 0.782235969951392,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9935483870967742,
-      "eval_linker_heavy_atoms_difference": 0.3836500220945647,
-      "eval_linker_heavy_atoms_difference_norm": 0.003922412538702962,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9935483870967742,
-      "eval_loss": 0.30979764461517334,
-      "eval_num_fragments": 2.999734865223155,
-      "eval_poi_equal": 0.7688024745912505,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9324790101634998,
-      "eval_poi_heavy_atoms_difference": 1.9796730004418912,
-      "eval_poi_heavy_atoms_difference_norm": 0.0652935121657418,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9324790101634998,
-      "eval_reassembly": 0.563323022536456,
-      "eval_reassembly_nostereo": 0.5840919133893062,
-      "eval_runtime": 1489.7161,
-      "eval_samples_per_second": 7.595,
-      "eval_steps_per_second": 0.119,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.919222271321255,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.10187249630689621,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0031,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.07569529116153717,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0034,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.07957209646701813,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0031,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.07916957885026932,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.0028,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.06219175085425377,
-      "learning_rate": 7.897721300792183e-05,
-      "loss": 0.003,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5534246575342465,
-      "eval_e3_equal": 0.8113124171453823,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9819708351745471,
-      "eval_e3_heavy_atoms_difference": 0.6880247459125055,
-      "eval_e3_heavy_atoms_difference_norm": 0.015301653091196767,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9819708351745471,
-      "eval_has_all_attachment_points": 0.9765797613787008,
-      "eval_has_three_substructures": 0.999116217410517,
-      "eval_heavy_atoms_difference": 8.727883340698188,
-      "eval_heavy_atoms_difference_norm": 0.1127822284643258,
-      "eval_linker_equal": 0.7893946089262042,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9933716305788776,
-      "eval_linker_heavy_atoms_difference": 0.420415377817057,
-      "eval_linker_heavy_atoms_difference_norm": 0.006358583874060475,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9933716305788776,
-      "eval_loss": 0.31512290239334106,
-      "eval_num_fragments": 3.000353513035793,
-      "eval_poi_equal": 0.7712770658418029,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9185152452496685,
-      "eval_poi_heavy_atoms_difference": 2.5423773751657093,
-      "eval_poi_heavy_atoms_difference_norm": 0.07831832886442663,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9185152452496685,
-      "eval_reassembly": 0.5626159964648696,
-      "eval_reassembly_nostereo": 0.5887759611135661,
-      "eval_runtime": 1512.8952,
-      "eval_samples_per_second": 7.479,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8976579761378701,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=7-learning_rate=7.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.920",
+  "trial_name": "trial-number=8-learning_rate=2.0e-05-warmup_ratio=0.020-min_lr=0.000-factor=0.820",
   "trial_params": {
-    "factor": 0.92,
-    "learning_rate": 7.897721300792183e-05,
-    "min_lr": 6.961786260568122e-10,
-    "warmup_ratio": 0.06
+    "factor": 0.8200000000000001,
+    "learning_rate": 1.9500595004156286e-05,
+    "min_lr": 1.163109026324453e-09,
+    "warmup_ratio": 0.02
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5469ff7a1aab9e7b2bbf77be4b0414864393190fed763ce1d4337b52ad4db63f
+oid sha256:041278698216885e2a0cc9e60e59109a8f1a4d27ade63a2779a5251e3e4da60c
 size 7480