Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e90f837dbdc83d6ff1d36ed88a5f758c7aff8f2af42b4ee3cc6e3199c492f1c2
+oid sha256:2b267e3b2170ac823a4bd1937ec3da6e1efea6fef9c533be7f6d20e42aa564ba
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:605bc0609d280aeda2a6da9481c65b0a49caab3e91345e0228bff53aa0221cbe
+oid sha256:73d8e5109d1d97589ffeda06ee520cf472e405a4adabb88b9d6e34ff49e2805d
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df275f21fc02ebd7a685cfb4f5d1da72997f650ce6e157c035f4dcd2286af318
+oid sha256:f44de6d4a1eaf99ac0c69cae1392ef8825ebeecf8a8bb1f460d0c1b7d3ec23b0
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d41bea1f14dd093aa9e67b0516dad8b66a8181aa8e30c8ef5a5acb02b15f4d9
+oid sha256:274e8bb901a6c141401f64b7cb6e710f494a233d1e70715f6675d29c5a0fe17f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25-rand-smiles/trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.40503756076005304,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25-rand-smiles/trial-number=19-learning_rate=4.1e-05-warmup_ratio=0.100-num_cycles=10/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.42685526609420776,
-      "learning_rate": 0.00019329472560247591,
-      "loss": 0.9723,
+      "grad_norm": 0.7256695628166199,
+      "learning_rate": 2.6272899690440636e-05,
+      "loss": 1.4831,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.35251015424728394,
-      "learning_rate": 0.00016587905554826487,
-      "loss": 0.1984,
+      "grad_norm": 0.6323962807655334,
+      "learning_rate": 8.2058780263589e-07,
+      "loss": 0.2913,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.27298179268836975,
-      "learning_rate": 0.00012386935099906144,
-      "loss": 0.1747,
+      "grad_norm": 0.5247291326522827,
+      "learning_rate": 2.5656115903974315e-05,
+      "loss": 0.199,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.3367314338684082,
-      "learning_rate": 7.660559187156851e-05,
-      "loss": 0.1656,
+      "grad_norm": 0.35064372420310974,
+      "learning_rate": 6.49292407753829e-07,
+      "loss": 0.1137,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.42077603936195374,
-      "learning_rate": 3.459588732236506e-05,
-      "loss": 0.1601,
+      "grad_norm": 0.2944692373275757,
+      "learning_rate": 2.503389658442746e-05,
+      "loss": 0.0897,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.4543251395225525,
-      "learning_rate": 7.180217268153993e-06,
-      "loss": 0.1557,
+      "grad_norm": 0.3109510540962219,
+      "learning_rate": 4.977412313142637e-07,
+      "loss": 0.058,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.3860662579536438,
-      "learning_rate": 0.00020002105983889256,
-      "loss": 0.1547,
+      "grad_norm": 0.26662477850914,
+      "learning_rate": 2.4406868253911588e-05,
+      "loss": 0.0507,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.25479811429977417,
-      "learning_rate": 0.00018456259840233827,
-      "loss": 0.1717,
+      "grad_norm": 0.21393170952796936,
+      "learning_rate": 3.6608687250542647e-07,
+      "loss": 0.0355,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.2642967104911804,
-      "learning_rate": 0.00015035620715297246,
-      "loss": 0.1599,
+      "grad_norm": 0.2200288325548172,
+      "learning_rate": 2.3775662276804576e-05,
+      "loss": 0.0334,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.29358983039855957,
-      "learning_rate": 0.0001050069623635349,
-      "loss": 0.1556,
+      "grad_norm": 0.2086140215396881,
+      "learning_rate": 2.5446189610677646e-07,
+      "loss": 0.0242,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.014317277949624393,
+      "eval_all_ligands_equal": 0.40503756076005304,
+      "eval_e3_equal": 0.7647370746796288,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 8.766858152894388,
-      "eval_e3_heavy_atoms_difference_norm": 0.24459577104288943,
+      "eval_e3_has_attachment_point(s)": 0.9977021652673442,
+      "eval_e3_heavy_atoms_difference": 0.038886433937251434,
+      "eval_e3_heavy_atoms_difference_norm": -0.00788608561722811,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 1.039593460008838,
-      "eval_heavy_atoms_difference_norm": -0.024715159299071657,
-      "eval_linker_equal": 0.0008837825894829872,
+      "eval_e3_valid": 0.9977021652673442,
+      "eval_has_all_attachment_points": 0.9896597437030491,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 4.397083517454706,
+      "eval_heavy_atoms_difference_norm": 0.05706137173108604,
+      "eval_linker_equal": 0.5899248784798939,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -3.746442775077331,
-      "eval_linker_heavy_atoms_difference_norm": -0.515199064951927,
+      "eval_linker_has_attachment_point(s)": 0.9977021652673442,
+      "eval_linker_heavy_atoms_difference": 0.5045514803358374,
+      "eval_linker_heavy_atoms_difference_norm": -0.0013503484876929224,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.4980059564113617,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.001590808661069377,
+      "eval_linker_valid": 0.9977021652673442,
+      "eval_loss": 0.2810657024383545,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.7160406539991162,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -3.9808219178082194,
-      "eval_poi_heavy_atoms_difference_norm": -0.28981238314404895,
+      "eval_poi_has_attachment_point(s)": 0.953866548828988,
+      "eval_poi_heavy_atoms_difference": 1.373044631020769,
+      "eval_poi_heavy_atoms_difference_norm": 0.04012878779602948,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1333.3368,
-      "eval_samples_per_second": 8.486,
-      "eval_steps_per_second": 0.133,
+      "eval_poi_valid": 0.953866548828988,
+      "eval_reassembly": 0.41122403888643394,
+      "eval_reassembly_nostereo": 0.4464869642068051,
+      "eval_runtime": 1561.695,
+      "eval_samples_per_second": 7.245,
+      "eval_steps_per_second": 0.113,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
+      "eval_valid": 0.9507733097657977,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.32659029960632324,
-      "learning_rate": 5.8597320935737426e-05,
-      "loss": 0.1523,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.4497728943824768,
-      "learning_rate": 2.1445496804393928e-05,
-      "loss": 0.1492,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.427729070186615,
-      "learning_rate": 1.8114216919283654e-06,
-      "loss": 0.1473,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.3337651193141937,
-      "learning_rate": 0.00019641462097038355,
-      "loss": 0.1602,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.26521092653274536,
-      "learning_rate": 0.000172782741406637,
-      "loss": 0.1574,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.016349977905435263,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 9.156606274856385,
-      "eval_e3_heavy_atoms_difference_norm": 0.25969747704928714,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 6.036411842686699,
-      "eval_heavy_atoms_difference_norm": 0.04351328557712477,
-      "eval_linker_equal": 0.001590808661069377,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -3.159787892178524,
-      "eval_linker_heavy_atoms_difference_norm": -0.471861120021933,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.488023579120636,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.001237295625276182,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": 0.03959346000883782,
-      "eval_poi_heavy_atoms_difference_norm": -0.14220090547078876,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1213.099,
-      "eval_samples_per_second": 9.327,
-      "eval_steps_per_second": 0.146,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.25984427332878113,
-      "learning_rate": 0.0001330219370657516,
-      "loss": 0.1524,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.35957634449005127,
-      "learning_rate": 8.606665290121947e-05,
-      "loss": 0.15,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.35782885551452637,
-      "learning_rate": 4.22496020327929e-05,
-      "loss": 0.1474,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.4192562997341156,
-      "learning_rate": 1.123046829039008e-05,
-      "loss": 0.1454,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.40985557436943054,
-      "learning_rate": 1.8169033799399541e-10,
-      "loss": 0.1443,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.006539991162174105,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -5.998144056562086,
-      "eval_e3_heavy_atoms_difference_norm": -0.3325506569989276,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -21.569509500662836,
-      "eval_heavy_atoms_difference_norm": -0.335015150576945,
-      "eval_linker_equal": 0.0014140521431727795,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -2.5268228015908085,
-      "eval_linker_heavy_atoms_difference_norm": -0.4262626647223243,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5094376802444458,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0011489173663278833,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -13.044542642509942,
-      "eval_poi_heavy_atoms_difference_norm": -0.6238375618341655,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1702.0989,
-      "eval_samples_per_second": 6.648,
-      "eval_steps_per_second": 0.104,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3",
+  "trial_name": "trial-number=19-learning_rate=4.1e-05-warmup_ratio=0.100-num_cycles=10",
   "trial_params": {
-    "learning_rate": 0.0002004749428706299,
-    "num_cycles": 3,
-    "warmup_ratio": 0.01
+    "learning_rate": 4.051582532010182e-05,
+    "num_cycles": 10,
+    "warmup_ratio": 0.09999999999999999
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0aa03ac7f5f501a4b5e63c42c61a196216b607ccab089627b34b398701de18ce
+oid sha256:ecba7819b717261821b6d8f873e7b174456ae3cf3d7471af2fac4191c2df62e6
 size 7544