Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f405a5f3acccefd6dd21b3fac6c730593e4e187227469c6a8467be4714b9bc34
+oid sha256:9f562dfe18e220ab10f689b9e2911afee519cd2efea1488ac824a686974524e1
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a44fb81d9d2f007de62d8e573b991ecf6924527b2c26b4d62da24eedba5469c
+oid sha256:33370b72853c47682c5381ebaede5916cd15882b17410fea394cde4e6a434d88
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4108544d3402968f4302901830233f225945a171703e572d09bf3b194d5d06b4
+oid sha256:f6551380c7d26c38ebd848f4def50edfca5bca6e3b37e3f1e5e3c1dccb160990
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b7b070ada8df300c245e6daa26454039d86686f1bc260ce873c14c0e22ed613
+oid sha256:f511ba4360276a988a88b86aba561d59c55b9c2982c0705456539fb5e3519ce4
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.4395050817498895,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25-rand-smiles/trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=6/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25-rand-smiles/trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.9023119211196899,
-      "learning_rate": 2.0006150861219307e-05,
-      "loss": 1.7674,
+      "grad_norm": 0.42685526609420776,
+      "learning_rate": 0.00019329472560247591,
+      "loss": 0.9723,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.7279019355773926,
-      "learning_rate": 9.95453959695377e-06,
-      "loss": 0.3955,
+      "grad_norm": 0.35251015424728394,
+      "learning_rate": 0.00016587905554826487,
+      "loss": 0.1984,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.6141806244850159,
-      "learning_rate": 1.2902076164106024e-06,
-      "loss": 0.2702,
+      "grad_norm": 0.27298179268836975,
+      "learning_rate": 0.00012386935099906144,
+      "loss": 0.1747,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.6135151386260986,
-      "learning_rate": 2.1922365111122713e-05,
-      "loss": 0.2385,
+      "grad_norm": 0.3367314338684082,
+      "learning_rate": 7.660559187156851e-05,
+      "loss": 0.1656,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.4508245885372162,
-      "learning_rate": 1.325803313057954e-05,
-      "loss": 0.1605,
+      "grad_norm": 0.42077603936195374,
+      "learning_rate": 3.459588732236506e-05,
+      "loss": 0.1601,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.5067362785339355,
-      "learning_rate": 3.206421866314002e-06,
-      "loss": 0.1223,
+      "grad_norm": 0.4543251395225525,
+      "learning_rate": 7.180217268153993e-06,
+      "loss": 0.1557,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.3615262806415558,
-      "learning_rate": 2.3002832012997965e-05,
-      "loss": 0.1114,
+      "grad_norm": 0.3860662579536438,
+      "learning_rate": 0.00020002105983889256,
+      "loss": 0.1547,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.34383073449134827,
-      "learning_rate": 1.642771196447441e-05,
-      "loss": 0.0913,
+      "grad_norm": 0.25479811429977417,
+      "learning_rate": 0.00018456259840233827,
+      "loss": 0.1717,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.342235267162323,
-      "learning_rate": 5.803143181883335e-06,
-      "loss": 0.0675,
+      "grad_norm": 0.2642967104911804,
+      "learning_rate": 0.00015035620715297246,
+      "loss": 0.1599,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.3004266023635864,
-      "learning_rate": 5.255416329408794e-08,
-      "loss": 0.0594,
+      "grad_norm": 0.29358983039855957,
+      "learning_rate": 0.0001050069623635349,
+      "loss": 0.1556,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.1444984533804684,
-      "eval_e3_equal": 0.6479893946089262,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.014317277949624393,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9978789217852408,
-      "eval_e3_heavy_atoms_difference": -0.34953601414052143,
-      "eval_e3_heavy_atoms_difference_norm": -0.02863605514454179,
+      "eval_e3_has_attachment_point(s)": 1.0,
+      "eval_e3_heavy_atoms_difference": 8.766858152894388,
+      "eval_e3_heavy_atoms_difference_norm": 0.24459577104288943,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9978789217852408,
-      "eval_has_all_attachment_points": 0.9886875828546178,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 5.697392841361025,
-      "eval_heavy_atoms_difference_norm": 0.0740994924263737,
-      "eval_linker_equal": 0.25046398585947854,
+      "eval_e3_valid": 1.0,
+      "eval_has_all_attachment_points": 1.0,
+      "eval_has_three_substructures": 1.0,
+      "eval_heavy_atoms_difference": 1.039593460008838,
+      "eval_heavy_atoms_difference_norm": -0.024715159299071657,
+      "eval_linker_equal": 0.0008837825894829872,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9961997348652232,
-      "eval_linker_heavy_atoms_difference": 1.2128148475475033,
-      "eval_linker_heavy_atoms_difference_norm": 0.025527521802678938,
+      "eval_linker_has_attachment_point(s)": 1.0,
+      "eval_linker_heavy_atoms_difference": -3.746442775077331,
+      "eval_linker_heavy_atoms_difference_norm": -0.515199064951927,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9961997348652232,
-      "eval_loss": 0.29746273159980774,
-      "eval_num_fragments": 2.99946973044631,
-      "eval_poi_equal": 0.6163499779054352,
+      "eval_linker_valid": 1.0,
+      "eval_loss": 0.4980059564113617,
+      "eval_num_fragments": 3.0,
+      "eval_poi_equal": 0.001590808661069377,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9383119752540875,
-      "eval_poi_heavy_atoms_difference": 1.7718073353954926,
-      "eval_poi_heavy_atoms_difference_norm": 0.0482882700406915,
+      "eval_poi_has_attachment_point(s)": 1.0,
+      "eval_poi_heavy_atoms_difference": -3.9808219178082194,
+      "eval_poi_heavy_atoms_difference_norm": -0.28981238314404895,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9383119752540875,
-      "eval_reassembly": 0.14670790985417587,
-      "eval_reassembly_nostereo": 0.16897923110914714,
-      "eval_runtime": 1576.9048,
-      "eval_samples_per_second": 7.175,
-      "eval_steps_per_second": 0.112,
+      "eval_poi_valid": 1.0,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 1333.3368,
+      "eval_samples_per_second": 8.486,
+      "eval_steps_per_second": 0.133,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.935218736190897,
+      "eval_valid": 1.0,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.324024498462677,
-      "learning_rate": 1.9206787570322047e-05,
-      "loss": 0.0568,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.2771834135055542,
-      "learning_rate": 8.870000645427467e-06,
-      "loss": 0.0441,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.2791912257671356,
-      "learning_rate": 8.313822822439741e-07,
-      "loss": 0.0372,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.29013025760650635,
-      "learning_rate": 2.137011577033572e-05,
-      "loss": 0.0375,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.28004390001296997,
-      "learning_rate": 1.2158535704553633e-05,
-      "loss": 0.0315,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.34653115333627926,
-      "eval_e3_equal": 0.7436146707909854,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9956694653115333,
-      "eval_e3_heavy_atoms_difference": 0.09244365885992047,
-      "eval_e3_heavy_atoms_difference_norm": -0.006706539383671118,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9956694653115333,
-      "eval_has_all_attachment_points": 0.9877154220061864,
-      "eval_has_three_substructures": 0.9992045956694653,
-      "eval_heavy_atoms_difference": 4.900927971718957,
-      "eval_heavy_atoms_difference_norm": 0.063723933664195,
-      "eval_linker_equal": 0.521608484312859,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9970835174547061,
-      "eval_linker_heavy_atoms_difference": 0.4762704374723818,
-      "eval_linker_heavy_atoms_difference_norm": -0.007590965723776292,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9970835174547061,
-      "eval_loss": 0.2942357063293457,
-      "eval_num_fragments": 3.000265134776845,
-      "eval_poi_equal": 0.6771542200618648,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9486522315510384,
-      "eval_poi_heavy_atoms_difference": 1.53610251878038,
-      "eval_poi_heavy_atoms_difference_norm": 0.042930097486257694,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9486522315510384,
-      "eval_reassembly": 0.3513919575784357,
-      "eval_reassembly_nostereo": 0.377817057003977,
-      "eval_runtime": 1525.7176,
-      "eval_samples_per_second": 7.416,
-      "eval_steps_per_second": 0.116,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9431727794962439,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.20582666993141174,
-      "learning_rate": 2.483129049056858e-06,
-      "loss": 0.0258,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.2565276026725769,
-      "learning_rate": 2.2742436579558516e-05,
-      "loss": 0.0251,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.24538569152355194,
-      "learning_rate": 1.540233080644256e-05,
-      "loss": 0.0245,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.2011946439743042,
-      "learning_rate": 4.873979764001823e-06,
-      "loss": 0.0195,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.1907787024974823,
-      "learning_rate": 0.0,
-      "loss": 0.0173,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.4395050817498895,
-      "eval_e3_equal": 0.771011931064958,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9971718957136544,
-      "eval_e3_heavy_atoms_difference": 0.08572691117984976,
-      "eval_e3_heavy_atoms_difference_norm": -0.0069188206629703436,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9971718957136544,
-      "eval_has_all_attachment_points": 0.9891294741493593,
-      "eval_has_three_substructures": 0.9992045956694653,
-      "eval_heavy_atoms_difference": 4.2998674326115776,
-      "eval_heavy_atoms_difference_norm": 0.05609787475305453,
-      "eval_linker_equal": 0.6337604949182502,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9975254087494476,
-      "eval_linker_heavy_atoms_difference": 0.4205037560760053,
-      "eval_linker_heavy_atoms_difference_norm": -0.0032995826470967765,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9975254087494476,
-      "eval_loss": 0.29654660820961,
-      "eval_num_fragments": 2.9995581087052585,
-      "eval_poi_equal": 0.7164825452938577,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9528943879805568,
-      "eval_poi_heavy_atoms_difference": 1.364825452938577,
-      "eval_poi_heavy_atoms_difference_norm": 0.039714887659171236,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9528943879805568,
-      "eval_reassembly": 0.44604507291206363,
-      "eval_reassembly_nostereo": 0.47892178524083073,
-      "eval_runtime": 1510.8484,
-      "eval_samples_per_second": 7.489,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9496243923994697,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=6",
+  "trial_name": "trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3",
   "trial_params": {
-    "learning_rate": 2.321257272753331e-05,
-    "num_cycles": 6,
-    "warmup_ratio": 0.06999999999999999
+    "learning_rate": 0.0002004749428706299,
+    "num_cycles": 3,
+    "warmup_ratio": 0.01
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:571d2bdb48d7ff0cb1195b9652496adcfe99bd07e801255079b9ff66a7c19573
+oid sha256:0aa03ac7f5f501a4b5e63c42c61a196216b607ccab089627b34b398701de18ce
 size 7544