Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e109d5d429d2aa62fd8941a16f02fc70c2b77f7e8516c2bd9f2b9c59bd6cced
+oid sha256:e6cbbdadb20e64f15020e12e55a078248311e1c567876699b4eaa4cb8702b5b5
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f49c5edb1ede90da01989bc6fa67f0a228cc55c814efe5f6e3df849a8f76ae4c
+oid sha256:758abb36ef13388f8f3eca1eb622dac5b2c61c10eea0fcb8626ab377f7e69eb2
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68c68b63cc38be0164d2f1e5e950d9224846ad3e331ae1b314dd8662e88bce2f
+oid sha256:e73cd41e4817e0862853abe9773b0e01081cac4bb2de2949d6cd33bd2894c93d
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98a089bb4a69e00a7e8a8814929d1de493c84d2bd6273a6f081d3b7a9500a40d
+oid sha256:a7c3ad366a455d36d790316ad7de46df5551299c0b50500acfaef90eebc7bbcf
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0010605391073795847,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=20-learning_rate=4.8e-06-warmup_ratio=0.030-num_cycles=3.000/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=1.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 2.0220000743865967,
-      "learning_rate": 3.4057266290417822e-06,
-      "loss": 4.8935,
+      "grad_norm": 4.175539493560791,
+      "learning_rate": 1.1556051821104526e-06,
+      "loss": 6.3946,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.25215482711792,
-      "learning_rate": 4.331843781954863e-06,
-      "loss": 1.2916,
+      "grad_norm": 1.8938024044036865,
+      "learning_rate": 1.5782537021726088e-06,
+      "loss": 2.3689,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.2481876611709595,
-      "learning_rate": 2.2556236723022006e-06,
-      "loss": 0.8525,
+      "grad_norm": 1.4588741064071655,
+      "learning_rate": 1.3637510062898264e-06,
+      "loss": 1.3991,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.2179811000823975,
-      "learning_rate": 2.9711989198042844e-07,
-      "loss": 0.7156,
+      "grad_norm": 1.4487122297286987,
+      "learning_rate": 1.0095554213221014e-06,
+      "loss": 1.164,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.2267446517944336,
-      "learning_rate": 3.0102179436748257e-07,
-      "loss": 0.6899,
+      "grad_norm": 1.3530123233795166,
+      "learning_rate": 6.046599668215422e-07,
+      "loss": 1.0414,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.3748468160629272,
-      "learning_rate": 2.263654228124041e-06,
-      "loss": 0.672,
+      "grad_norm": 1.4081937074661255,
+      "learning_rate": 2.5079620192718024e-07,
+      "loss": 0.9813,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.3039966821670532,
-      "learning_rate": 4.336439114656795e-06,
-      "loss": 0.5886,
+      "grad_norm": 1.3146899938583374,
+      "learning_rate": 3.687377510596774e-08,
+      "loss": 0.9546,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.32431960105896,
-      "learning_rate": 4.567047206277767e-06,
-      "loss": 0.4805,
+      "grad_norm": 1.405544638633728,
+      "learning_rate": 1.664152886385891e-08,
+      "loss": 0.9509,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.9057373404502869,
-      "learning_rate": 2.7382717272379126e-06,
-      "loss": 0.3947,
+      "grad_norm": 1.4300780296325684,
+      "learning_rate": 1.9518289383020806e-07,
+      "loss": 0.9464,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.8103320598602295,
-      "learning_rate": 5.726126235670513e-07,
-      "loss": 0.3541,
+      "grad_norm": 1.3519362211227417,
+      "learning_rate": 5.276386570231308e-07,
+      "loss": 0.9309,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.12311091471498012,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.023862129916040652,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.989924878479894,
-      "eval_e3_heavy_atoms_difference": -1.5570481661511268,
-      "eval_e3_heavy_atoms_difference_norm": -0.08974618004920261,
+      "eval_e3_has_attachment_point(s)": 0.8015024304021211,
+      "eval_e3_heavy_atoms_difference": 6.527529827662395,
+      "eval_e3_heavy_atoms_difference_norm": 0.17258164424657527,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.989924878479894,
-      "eval_has_all_attachment_points": 0.8783915156871409,
-      "eval_has_three_substructures": 0.9910737958462218,
-      "eval_heavy_atoms_difference": 39.06195315952276,
-      "eval_heavy_atoms_difference_norm": 0.4968459886092445,
-      "eval_linker_equal": 0.0023862129916040653,
+      "eval_e3_valid": 0.8015024304021211,
+      "eval_has_all_attachment_points": 0.4949182501104728,
+      "eval_has_three_substructures": 0.8086610693769333,
+      "eval_heavy_atoms_difference": 71.589748121962,
+      "eval_heavy_atoms_difference_norm": 0.9393652926517526,
+      "eval_linker_equal": 0.0007954043305346885,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9831197525408749,
-      "eval_linker_heavy_atoms_difference": 6.709854175872735,
-      "eval_linker_heavy_atoms_difference_norm": 0.29386451426862076,
+      "eval_linker_has_attachment_point(s)": 0.723376049491825,
+      "eval_linker_heavy_atoms_difference": 11.69456473707468,
+      "eval_linker_heavy_atoms_difference_norm": 0.6508096279718948,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9831197525408749,
-      "eval_loss": 0.4712540805339813,
-      "eval_num_fragments": 2.995669465311533,
-      "eval_poi_equal": 0.04657534246575343,
+      "eval_linker_valid": 0.723376049491825,
+      "eval_loss": 0.9551888108253479,
+      "eval_num_fragments": 3.1498895271763145,
+      "eval_poi_equal": 0.0,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.5917808219178082,
-      "eval_poi_heavy_atoms_difference": 16.23835616438356,
-      "eval_poi_heavy_atoms_difference_norm": 0.4633671091597373,
+      "eval_poi_has_attachment_point(s)": 0.1647370746796288,
+      "eval_poi_heavy_atoms_difference": 29.040653999116216,
+      "eval_poi_heavy_atoms_difference_norm": 0.9184080533333931,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.5917808219178082,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 0.00017675651789659743,
-      "eval_runtime": 1477.0223,
-      "eval_samples_per_second": 7.661,
-      "eval_steps_per_second": 0.12,
+      "eval_poi_valid": 0.1647370746796288,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 2290.4515,
+      "eval_samples_per_second": 4.94,
+      "eval_steps_per_second": 0.077,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.5875386654882899,
+      "eval_valid": 0.1528060097216085,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.791894793510437,
-      "learning_rate": 1.098761656065621e-07,
-      "loss": 0.3429,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.6823320984840393,
-      "learning_rate": 1.785907831928506e-06,
-      "loss": 0.339,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.739353597164154,
-      "learning_rate": 4.02207509608434e-06,
-      "loss": 0.3174,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.9243476390838623,
-      "learning_rate": 4.712160964261165e-06,
-      "loss": 0.2791,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.6906940340995789,
-      "learning_rate": 3.206182491961818e-06,
-      "loss": 0.2429,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.000618647812638091,
-      "eval_e3_equal": 0.1857711003093239,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9948740609809986,
-      "eval_e3_heavy_atoms_difference": -1.1957578435704816,
-      "eval_e3_heavy_atoms_difference_norm": -0.0694718436129326,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9948740609809986,
-      "eval_has_all_attachment_points": 0.9208130799823243,
-      "eval_has_three_substructures": 0.9956694653115333,
-      "eval_heavy_atoms_difference": 27.105612019443218,
-      "eval_heavy_atoms_difference_norm": 0.34485278223335736,
-      "eval_linker_equal": 0.0043305346884666375,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.988775961113566,
-      "eval_linker_heavy_atoms_difference": 5.574723817940787,
-      "eval_linker_heavy_atoms_difference_norm": 0.23041477586088768,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.988775961113566,
-      "eval_loss": 0.39499878883361816,
-      "eval_num_fragments": 2.9969067609368096,
-      "eval_poi_equal": 0.1516570923552806,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.7239946973044631,
-      "eval_poi_heavy_atoms_difference": 10.551303579319487,
-      "eval_poi_heavy_atoms_difference_norm": 0.29845879597550384,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.7239946973044631,
-      "eval_reassembly": 0.000618647812638091,
-      "eval_reassembly_nostereo": 0.0007070260715863897,
-      "eval_runtime": 1486.4004,
-      "eval_samples_per_second": 7.612,
-      "eval_steps_per_second": 0.119,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.7191338930623067,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.7934216260910034,
-      "learning_rate": 9.226013039121686e-07,
-      "loss": 0.2229,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.6689370274543762,
-      "learning_rate": 1.2292954959110304e-08,
-      "loss": 0.2152,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.5731592178344727,
-      "learning_rate": 1.3326650929596504e-06,
-      "loss": 0.2145,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.7813031673431396,
-      "learning_rate": 3.640076296365813e-06,
-      "loss": 0.207,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.7383819818496704,
-      "learning_rate": 4.7612058274004114e-06,
-      "loss": 0.191,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0010605391073795847,
-      "eval_e3_equal": 0.2532920901458241,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9965532479010164,
-      "eval_e3_heavy_atoms_difference": -1.2925320371188687,
-      "eval_e3_heavy_atoms_difference_norm": -0.06944638367108885,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9965532479010164,
-      "eval_has_all_attachment_points": 0.9456473707467963,
-      "eval_has_three_substructures": 0.9971718957136544,
-      "eval_heavy_atoms_difference": 20.10534688466637,
-      "eval_heavy_atoms_difference_norm": 0.2561670560292284,
-      "eval_linker_equal": 0.00901458241272647,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.989924878479894,
-      "eval_linker_heavy_atoms_difference": 4.3442333186036235,
-      "eval_linker_heavy_atoms_difference_norm": 0.15665268892474063,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.989924878479894,
-      "eval_loss": 0.36527007818222046,
-      "eval_num_fragments": 2.997702165267344,
-      "eval_poi_equal": 0.24498453380468405,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.7915156871409633,
-      "eval_poi_heavy_atoms_difference": 7.419708351745471,
-      "eval_poi_heavy_atoms_difference_norm": 0.20887258408048587,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.7915156871409633,
-      "eval_reassembly": 0.0010605391073795847,
-      "eval_reassembly_nostereo": 0.001590808661069377,
-      "eval_runtime": 1518.7362,
-      "eval_samples_per_second": 7.45,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.785771100309324,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=20-learning_rate=4.8e-06-warmup_ratio=0.030-num_cycles=3.000",
+  "trial_name": "trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=1.500",
   "trial_params": {
-    "learning_rate": 4.7612058274004114e-06,
-    "num_cycles": 3.0,
-    "warmup_ratio": 0.03
+    "learning_rate": 1.6155360445904128e-06,
+    "num_cycles": 1.5,
+    "warmup_ratio": 0.08
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81263aba5865c96cbbde9084dc101d24a821edb1f7e92f8ff71ce2ce065e533e
+oid sha256:cbefcfb3ae3485c21410a4c0d3adfd89f163e7aa4441b2c14560931831d9b0ab
 size 7480