Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e012f5a2962b1055e7e91314208508fb0e2b5839688d9c61c30ead3cb378e53
+oid sha256:ef49e8f550cab783de7a0ba94f48c1f7540809bf316b953689dd80be1a5dd39a
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b28a919b0dae87b17bfb875069130ea193e1bbe0b825ed2e91ca448339d232fd
+oid sha256:1fa9696a1cc6991b07a657bebe540a630458ccc038ac0c9a9107832507d142ed
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df90c14ac5ab218269cb46bc8c052aca82e45e962bacfffe6b5d990927a38a8d
+oid sha256:da000c9e378f4a1d47259777e81cf5543a01e26fac996b3222d393e65ab9bfec
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d92c5a2febec6a3debc54e07f631908af44ee341af8b138c2ac3072c6cfbde8
+oid sha256:2e531f97d95afebd39a8f25f638079450f8fd8f16cc178cd6d0e1cd555754a19
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=2/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.4987185152452497,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 2.753248691558838,
-      "learning_rate": 1.5896460305053544e-06,
-      "loss": 4.3638,
+      "grad_norm": 0.6311352849006653,
+      "learning_rate": 6.321657849706081e-05,
+      "loss": 1.1975,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.679311990737915,
-      "learning_rate": 1.4873057247591052e-06,
-      "loss": 1.6493,
+      "grad_norm": 0.3875294029712677,
+      "learning_rate": 2.266437844344336e-05,
+      "loss": 0.1246,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.3882453441619873,
-      "learning_rate": 1.3171083031390388e-06,
-      "loss": 1.2617,
+      "grad_norm": 0.31023937463760376,
+      "learning_rate": 1.9519067734396335e-08,
+      "loss": 0.0683,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.3434231281280518,
-      "learning_rate": 1.0960492996948814e-06,
-      "loss": 1.0781,
+      "grad_norm": 0.28427359461784363,
+      "learning_rate": 5.707999615476073e-05,
+      "loss": 0.0528,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.424155592918396,
-      "learning_rate": 8.462031723371678e-07,
-      "loss": 0.9639,
+      "grad_norm": 0.22120411694049835,
+      "learning_rate": 1.6281206903827376e-05,
+      "loss": 0.0246,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.3910627365112305,
-      "learning_rate": 5.925189968173456e-07,
-      "loss": 0.8928,
+      "grad_norm": 0.25376608967781067,
+      "learning_rate": 7.705306860082544e-05,
+      "loss": 0.0182,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.392408847808838,
-      "learning_rate": 3.603291077964441e-07,
-      "loss": 0.8463,
+      "grad_norm": 0.1665523201227188,
+      "learning_rate": 5.028164398882772e-05,
+      "loss": 0.0197,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.34941828250885,
-      "learning_rate": 1.7281946853579862e-07,
-      "loss": 0.8216,
+      "grad_norm": 0.1313042938709259,
+      "learning_rate": 1.0710840084315767e-05,
+      "loss": 0.0104,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.3188016414642334,
-      "learning_rate": 4.871437243757388e-08,
-      "loss": 0.8072,
+      "grad_norm": 0.2163771092891693,
+      "learning_rate": 7.52019442839167e-05,
+      "loss": 0.0097,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 1.4827531576156616,
-      "learning_rate": 4.066769634440437e-10,
-      "loss": 0.8052,
+      "grad_norm": 0.12024851143360138,
+      "learning_rate": 4.306723215947234e-05,
+      "loss": 0.0118,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.02898806893504198,
+      "eval_all_ligands_equal": 0.4987185152452497,
+      "eval_e3_equal": 0.7911621741051701,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.8969509500662837,
-      "eval_e3_heavy_atoms_difference": 3.6503756076005303,
-      "eval_e3_heavy_atoms_difference_norm": 0.07527027809161804,
+      "eval_e3_has_attachment_point(s)": 0.9909854175872735,
+      "eval_e3_heavy_atoms_difference": 0.5697746354396819,
+      "eval_e3_heavy_atoms_difference_norm": 0.013845663202264,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.8969509500662837,
-      "eval_has_all_attachment_points": 0.5550154661953159,
-      "eval_has_three_substructures": 0.9007512152010605,
-      "eval_heavy_atoms_difference": 69.17551922227132,
-      "eval_heavy_atoms_difference_norm": 0.9045006813289546,
-      "eval_linker_equal": 0.0011489173663278833,
+      "eval_e3_valid": 0.9909854175872735,
+      "eval_has_all_attachment_points": 0.9860362350861688,
+      "eval_has_three_substructures": 0.9984091913389306,
+      "eval_heavy_atoms_difference": 6.771895713654441,
+      "eval_heavy_atoms_difference_norm": 0.0888871888240822,
+      "eval_linker_equal": 0.7210782147591692,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.8592134334953602,
-      "eval_linker_heavy_atoms_difference": 10.46796288113124,
-      "eval_linker_heavy_atoms_difference_norm": 0.5573060373612391,
+      "eval_linker_has_attachment_point(s)": 0.9970835174547061,
+      "eval_linker_heavy_atoms_difference": 0.5161290322580645,
+      "eval_linker_heavy_atoms_difference_norm": 0.006289913322434787,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.8592134334953602,
-      "eval_loss": 0.856566846370697,
-      "eval_num_fragments": 3.071056120194432,
-      "eval_poi_equal": 0.0,
+      "eval_linker_valid": 0.9970835174547061,
+      "eval_loss": 0.2860860526561737,
+      "eval_num_fragments": 3.0001767565178965,
+      "eval_poi_equal": 0.7388422448077773,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.21140079540433054,
-      "eval_poi_heavy_atoms_difference": 27.802297834732656,
-      "eval_poi_heavy_atoms_difference_norm": 0.8689823403774681,
+      "eval_poi_has_attachment_point(s)": 0.9325673884224481,
+      "eval_poi_heavy_atoms_difference": 2.120724701723376,
+      "eval_poi_heavy_atoms_difference_norm": 0.06647977641739568,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.21140079540433054,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1718.3667,
-      "eval_samples_per_second": 6.585,
-      "eval_steps_per_second": 0.103,
+      "eval_poi_valid": 0.9325673884224481,
+      "eval_reassembly": 0.5070260715863898,
+      "eval_reassembly_nostereo": 0.5330976579761378,
+      "eval_runtime": 1575.5066,
+      "eval_samples_per_second": 7.182,
+      "eval_steps_per_second": 0.112,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.19938135218736192,
+      "eval_valid": 0.9245249668581529,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 1.1991398334503174,
-      "learning_rate": 1.5828157639979357e-06,
-      "loss": 0.7666,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 1.2663832902908325,
-      "learning_rate": 1.473107625487659e-06,
-      "loss": 0.6824,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 1.2193111181259155,
-      "learning_rate": 1.296960161557218e-06,
-      "loss": 0.6184,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 1.3224248886108398,
-      "learning_rate": 1.071963064180241e-06,
-      "loss": 0.5711,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 1.1388825178146362,
-      "learning_rate": 8.205840405238564e-07,
-      "loss": 0.5359,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.07414935925762263,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9784357048166151,
-      "eval_e3_heavy_atoms_difference": -0.01643835616438356,
-      "eval_e3_heavy_atoms_difference_norm": -0.041997529417292574,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9784357048166151,
-      "eval_has_all_attachment_points": 0.7959346000883782,
-      "eval_has_three_substructures": 0.9798497569597879,
-      "eval_heavy_atoms_difference": 59.51330092797172,
-      "eval_heavy_atoms_difference_norm": 0.7667716663424441,
-      "eval_linker_equal": 0.0017675651789659744,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9551922227132126,
-      "eval_linker_heavy_atoms_difference": 7.438002651347769,
-      "eval_linker_heavy_atoms_difference_norm": 0.3411107847976279,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9551922227132126,
-      "eval_loss": 0.6140255928039551,
-      "eval_num_fragments": 3.007423773751657,
-      "eval_poi_equal": 0.005921343349536014,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.3249668581528944,
-      "eval_poi_heavy_atoms_difference": 24.587361908970394,
-      "eval_poi_heavy_atoms_difference_norm": 0.7407628377914883,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.3249668581528944,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1405.5464,
-      "eval_samples_per_second": 8.05,
-      "eval_steps_per_second": 0.126,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.3149801148917366,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 1.2519776821136475,
-      "learning_rate": 5.679252379371353e-07,
-      "loss": 0.5131,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 1.2744039297103882,
-      "learning_rate": 3.392165996322104e-07,
-      "loss": 0.4951,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.9478299617767334,
-      "learning_rate": 1.5729645895619162e-07,
-      "loss": 0.4866,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 1.2296202182769775,
-      "learning_rate": 4.033095447780681e-08,
-      "loss": 0.4822,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 1.018215537071228,
-      "learning_rate": 0.0,
-      "loss": 0.4806,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.08387096774193549,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9821475916924437,
-      "eval_e3_heavy_atoms_difference": -0.6608926204153778,
-      "eval_e3_heavy_atoms_difference_norm": -0.06311356185857149,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9821475916924437,
-      "eval_has_all_attachment_points": 0.7999116217410517,
-      "eval_has_three_substructures": 0.9835616438356164,
-      "eval_heavy_atoms_difference": 55.602032699955814,
-      "eval_heavy_atoms_difference_norm": 0.7141083957383162,
-      "eval_linker_equal": 0.0016791869200176756,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9660627485638533,
-      "eval_linker_heavy_atoms_difference": 7.421387538665488,
-      "eval_linker_heavy_atoms_difference_norm": 0.34276215918626973,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9660627485638533,
-      "eval_loss": 0.5785325169563293,
-      "eval_num_fragments": 3.004065399911622,
-      "eval_poi_equal": 0.011135660627485638,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.3761378700839593,
-      "eval_poi_heavy_atoms_difference": 22.859920459566947,
-      "eval_poi_heavy_atoms_difference_norm": 0.6843588799912407,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.3761378700839593,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1402.5909,
-      "eval_samples_per_second": 8.067,
-      "eval_steps_per_second": 0.126,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.36897923110914715,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=2",
+  "trial_name": "trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7",
   "trial_params": {
-    "learning_rate": 1.6155360445904128e-06,
-    "num_cycles": 2,
-    "warmup_ratio": 0.08
+    "learning_rate": 7.754006328430222e-05,
+    "num_cycles": 7,
+    "warmup_ratio": 0.05
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee442d952c8ba0ce1c0c5a21c9b9694d76dd3f4d0fe5940b17559e07dd4f5a13
+oid sha256:ba024ca1598855de20b7dedcadbcfc630c71b61849b062104d468853b275667a
 size 7480