Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18deac3ef5df772e51d88f02658b10208faedb86ef50aa81032541a0003fbb1b
+oid sha256:83581d38161b27a5acb790f575f33ed3c02a7f1d7c2437ee178aa966f81080ae
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53cc8363c325a40480d2f4292cc117af2059d88e97388ded7bccafd2c737c6bc
+oid sha256:dffa6ac9b8d89e7a91077ceb9acc3c19df6aae53480883e9198e93f6f29bffc1
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:476f4c43b968c51e5cd7cb7f664df17fc7ffb68f277f27852fc77b84381ee676
+oid sha256:8ab19ef495d430a2accb0aebafeb69cc818c2a103c6f73355b5b8061f940d1b3
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16cf5f20a28dfa325b9f37e9a560f2177d69ef5b38261444003d24ebf40416c1
+oid sha256:f430e9f6ffa976be8b034917a3194b833ba551ae167ef81aa406b2df6c359139
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.00830755634114008,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=8-learning_rate=6.7e-06-warmup_ratio=0.040-num_cycles=7/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.001237295625276182,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.165015459060669,
-      "learning_rate": 5.430126458869007e-06,
-      "loss": 2.6784,
+      "grad_norm": 1.1444038152694702,
+      "learning_rate": 8.921312073289648e-06,
+      "loss": 2.3977,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.2574317455291748,
-      "learning_rate": 1.946806423022153e-06,
-      "loss": 0.9019,
+      "grad_norm": 1.1081920862197876,
+      "learning_rate": 8.346964207338686e-06,
+      "loss": 0.6966,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.2324624061584473,
-      "learning_rate": 1.6766330712113717e-09,
-      "loss": 0.7583,
+      "grad_norm": 0.9350355267524719,
+      "learning_rate": 7.391792877870348e-06,
+      "loss": 0.4502,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.2702449560165405,
-      "learning_rate": 4.9030112790194464e-06,
-      "loss": 0.6346,
+      "grad_norm": 0.7580628395080566,
+      "learning_rate": 6.151179358577131e-06,
+      "loss": 0.3328,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.0283399820327759,
-      "learning_rate": 1.3985099240210261e-06,
-      "loss": 0.4866,
+      "grad_norm": 0.879012405872345,
+      "learning_rate": 4.749008542126604e-06,
+      "loss": 0.2676,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.309403419494629,
-      "learning_rate": 6.618642079242619e-06,
-      "loss": 0.4472,
+      "grad_norm": 0.7280972003936768,
+      "learning_rate": 3.3252980717220435e-06,
+      "loss": 0.2305,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.7636657357215881,
-      "learning_rate": 4.3190519308453175e-06,
-      "loss": 0.3794,
+      "grad_norm": 0.47845959663391113,
+      "learning_rate": 2.022216492259077e-06,
+      "loss": 0.2078,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.7996349930763245,
-      "learning_rate": 9.200310665541939e-07,
-      "loss": 0.3196,
+      "grad_norm": 0.5540239810943604,
+      "learning_rate": 9.698866172476045e-07,
+      "loss": 0.1961,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.9124681353569031,
-      "learning_rate": 6.4596357019460475e-06,
-      "loss": 0.2998,
+      "grad_norm": 0.6173533201217651,
+      "learning_rate": 2.733917555418904e-07,
+      "loss": 0.189,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.785952091217041,
-      "learning_rate": 3.699354226680869e-06,
-      "loss": 0.259,
+      "grad_norm": 0.6460208296775818,
+      "learning_rate": 2.28232703021863e-09,
+      "loss": 0.1878,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.00035351303579319487,
-      "eval_e3_equal": 0.19734865223155104,
+      "eval_all_ligands_equal": 0.001237295625276182,
+      "eval_e3_equal": 0.2914714980114892,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.989041095890411,
-      "eval_e3_heavy_atoms_difference": -1.1672116659301812,
-      "eval_e3_heavy_atoms_difference_norm": -0.06655288029056082,
+      "eval_e3_has_attachment_point(s)": 0.993813521873619,
+      "eval_e3_heavy_atoms_difference": -1.2530269553689792,
+      "eval_e3_heavy_atoms_difference_norm": -0.06816270339643937,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.989041095890411,
-      "eval_has_all_attachment_points": 0.9113566062748564,
-      "eval_has_three_substructures": 0.9907202828104287,
-      "eval_heavy_atoms_difference": 31.27892178524083,
-      "eval_heavy_atoms_difference_norm": 0.39965092893478266,
-      "eval_linker_equal": 0.005125939019001326,
+      "eval_e3_valid": 0.993813521873619,
+      "eval_has_all_attachment_points": 0.94025629695095,
+      "eval_has_three_substructures": 0.9953159522757402,
+      "eval_heavy_atoms_difference": 23.324701723376048,
+      "eval_heavy_atoms_difference_norm": 0.3003596358168922,
+      "eval_linker_equal": 0.00901458241272647,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9786124613345117,
-      "eval_linker_heavy_atoms_difference": 5.976844896155546,
-      "eval_linker_heavy_atoms_difference_norm": 0.25897655372201456,
+      "eval_linker_has_attachment_point(s)": 0.9865665046398586,
+      "eval_linker_heavy_atoms_difference": 5.095183384887318,
+      "eval_linker_heavy_atoms_difference_norm": 0.21271914204862793,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9786124613345117,
-      "eval_loss": 0.4048418700695038,
-      "eval_num_fragments": 2.9939019001325673,
-      "eval_poi_equal": 0.1444100751215201,
+      "eval_linker_valid": 0.9865665046398586,
+      "eval_loss": 0.3655618727207184,
+      "eval_num_fragments": 2.997613787008396,
+      "eval_poi_equal": 0.2577110030932391,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.6709677419354839,
-      "eval_poi_heavy_atoms_difference": 12.235262925320372,
-      "eval_poi_heavy_atoms_difference_norm": 0.3551644875381415,
+      "eval_poi_has_attachment_point(s)": 0.7584622182942996,
+      "eval_poi_heavy_atoms_difference": 8.438621299160406,
+      "eval_poi_heavy_atoms_difference_norm": 0.24540271262613053,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.6709677419354839,
-      "eval_reassembly": 0.00035351303579319487,
-      "eval_reassembly_nostereo": 0.000618647812638091,
-      "eval_runtime": 1479.183,
-      "eval_samples_per_second": 7.649,
-      "eval_steps_per_second": 0.12,
+      "eval_poi_valid": 0.7584622182942996,
+      "eval_reassembly": 0.001237295625276182,
+      "eval_reassembly_nostereo": 0.0021210782147591694,
+      "eval_runtime": 1537.2247,
+      "eval_samples_per_second": 7.361,
+      "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.6633672116659302,
+      "eval_valid": 0.7521873619089704,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.5575995445251465,
-      "learning_rate": 5.286633231950076e-07,
-      "loss": 0.2284,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.6379813551902771,
-      "learning_rate": 6.187524693934964e-06,
-      "loss": 0.2183,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.4957185387611389,
-      "learning_rate": 3.0663156561612296e-06,
-      "loss": 0.1913,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.6114542484283447,
-      "learning_rate": 2.3855174388717685e-07,
-      "loss": 0.1752,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.5232323408126831,
-      "learning_rate": 5.8121438560172e-06,
-      "loss": 0.1682,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0020326999558108703,
-      "eval_e3_equal": 0.3287671232876712,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9969951391957579,
-      "eval_e3_heavy_atoms_difference": -0.8549712770658418,
-      "eval_e3_heavy_atoms_difference_norm": -0.04947634143007919,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9969951391957579,
-      "eval_has_all_attachment_points": 0.9474149359257623,
-      "eval_has_three_substructures": 0.9977905435262925,
-      "eval_heavy_atoms_difference": 20.506319045514804,
-      "eval_heavy_atoms_difference_norm": 0.2649850780773329,
-      "eval_linker_equal": 0.011842686699072028,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9893946089262041,
-      "eval_linker_heavy_atoms_difference": 4.9024304021210785,
-      "eval_linker_heavy_atoms_difference_norm": 0.20275821480182482,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9893946089262041,
-      "eval_loss": 0.356844037771225,
-      "eval_num_fragments": 2.9995581087052585,
-      "eval_poi_equal": 0.3149801148917366,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.7983208130799824,
-      "eval_poi_heavy_atoms_difference": 7.114007954043306,
-      "eval_poi_heavy_atoms_difference_norm": 0.20884483744925436,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.7983208130799824,
-      "eval_reassembly": 0.0021210782147591694,
-      "eval_reassembly_nostereo": 0.003535130357931949,
-      "eval_runtime": 1469.7686,
-      "eval_samples_per_second": 7.698,
-      "eval_steps_per_second": 0.12,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.7922227132125497,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.5564959645271301,
-      "learning_rate": 2.4428158825908893e-06,
-      "loss": 0.1505,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.47829896211624146,
-      "learning_rate": 6.018171698523952e-08,
-      "loss": 0.1412,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.690832257270813,
-      "learning_rate": 5.347060429720348e-06,
-      "loss": 0.1358,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.4241408109664917,
-      "learning_rate": 1.8513898122995003e-06,
-      "loss": 0.1226,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.4954194128513336,
-      "learning_rate": 0.0,
-      "loss": 0.1165,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.00830755634114008,
-      "eval_e3_equal": 0.4395050817498895,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9971718957136544,
-      "eval_e3_heavy_atoms_difference": -0.8420680512593902,
-      "eval_e3_heavy_atoms_difference_norm": -0.04755230561075204,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9971718957136544,
-      "eval_has_all_attachment_points": 0.9688908528501988,
-      "eval_has_three_substructures": 0.9978789217852408,
-      "eval_heavy_atoms_difference": 13.182147591692443,
-      "eval_heavy_atoms_difference_norm": 0.17133339381029822,
-      "eval_linker_equal": 0.0276623950508175,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9921343349536014,
-      "eval_linker_heavy_atoms_difference": 3.0407423773751656,
-      "eval_linker_heavy_atoms_difference_norm": 0.1076580313889189,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9921343349536014,
-      "eval_loss": 0.33115315437316895,
-      "eval_num_fragments": 2.998585947856827,
-      "eval_poi_equal": 0.4557666813963765,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8638091029606717,
-      "eval_poi_heavy_atoms_difference": 4.40415377817057,
-      "eval_poi_heavy_atoms_difference_norm": 0.1353384851123875,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8638091029606717,
-      "eval_reassembly": 0.008484312859036678,
-      "eval_reassembly_nostereo": 0.012196199734865224,
-      "eval_runtime": 1564.3499,
-      "eval_samples_per_second": 7.233,
-      "eval_steps_per_second": 0.113,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8590366769774636,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=8-learning_rate=6.7e-06-warmup_ratio=0.040-num_cycles=7",
+  "trial_name": "trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2",
   "trial_params": {
-    "learning_rate": 6.660473554133319e-06,
-    "num_cycles": 7,
-    "warmup_ratio": 0.04
+    "learning_rate": 9.066610391784642e-06,
+    "num_cycles": 2,
+    "warmup_ratio": 0.060000000000000005
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3112b91e46dbf9197faac613df90c3054d234a4c43d53e47cf611e6efc87b11
+oid sha256:4f4dd62486422c15e6a55bb71f34dd6dc931136ad9b366ebdee4dcb75d7c3cb7
 size 7480