Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0170f45558525269d3a5a8cb0b197b7ee3df057f50999a5698b42e1d2d35752b
+oid sha256:fa62b5efb1844ae53dd7f45956a6677f10bce73ad71356720907215536556f68
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62dd74ff886a6c83524f9e3f3e97fef8b4a3fb2298f99e72f93c9857619ee15c
+oid sha256:fc04c3f034f1b0d3fdd074893ec0e5aa7b2b5e5044a133b359c4677f487fa79c
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d94d536ee47bd1ac32787c28bada8e52b6c109c576acddc86c2efef15e3a3d23
+oid sha256:7944a123a20e716aab97dbd8cc4190b23fa0da33c6b81e437bd160e8a6452078
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:599f48ff68e6c3b32afae7e87ae636002b9267aee745003943d32e88902d20a7
+oid sha256:e485a1d5071b23b811f4a1b96aeaa0547c3f98de0681f55bcfba2da94aaa8a49
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5305346884666372,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=11-learning_rate=1.0e-04-warmup_ratio=0.080-num_cycles=3.000/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=12-learning_rate=2.2e-06-warmup_ratio=0.050-num_cycles=10.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.0087827444076538,
-      "learning_rate": 7.432336402923124e-05,
-      "loss": 1.8581,
+      "grad_norm": 4.169440746307373,
+      "learning_rate": 1.5613799171014363e-06,
+      "loss": 5.9987,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.4915059804916382,
-      "learning_rate": 9.453407081430295e-05,
-      "loss": 0.1818,
+      "grad_norm": 1.811974287033081,
+      "learning_rate": 6.043608340530767e-07,
+      "loss": 1.9745,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.2095542550086975,
-      "learning_rate": 4.922460243282525e-05,
-      "loss": 0.0556,
+      "grad_norm": 1.5660574436187744,
+      "learning_rate": 1.7934007606188836e-06,
+      "loss": 1.4372,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.19221505522727966,
-      "learning_rate": 6.484064135881745e-06,
-      "loss": 0.0311,
+      "grad_norm": 1.517979383468628,
+      "learning_rate": 2.134062284657905e-07,
+      "loss": 1.2106,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.17544284462928772,
-      "learning_rate": 6.569215571421697e-06,
-      "loss": 0.0257,
+      "grad_norm": 1.333193063735962,
+      "learning_rate": 2.0966891974916694e-06,
+      "loss": 1.1,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.2089957296848297,
-      "learning_rate": 4.939985370478998e-05,
-      "loss": 0.0256,
+      "grad_norm": 1.3593288660049438,
+      "learning_rate": 1.4611952262523042e-08,
+      "loss": 0.9629,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.20172426104545593,
-      "learning_rate": 9.463435501865708e-05,
-      "loss": 0.0263,
+      "grad_norm": 1.2619653940200806,
+      "learning_rate": 2.1799591909569794e-06,
+      "loss": 0.9001,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.1847366988658905,
-      "learning_rate": 9.9666928389484e-05,
-      "loss": 0.0205,
+      "grad_norm": 1.375174641609192,
+      "learning_rate": 5.1486486192622186e-08,
+      "loss": 0.801,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.13189482688903809,
-      "learning_rate": 5.975745811745197e-05,
-      "loss": 0.0124,
+      "grad_norm": 1.445817470550537,
+      "learning_rate": 2.024986117163073e-06,
+      "loss": 0.7564,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.11474896222352982,
-      "learning_rate": 1.2496157532491428e-05,
-      "loss": 0.0074,
+      "grad_norm": 1.337365746498108,
+      "learning_rate": 3.159594019707261e-07,
+      "loss": 0.6855,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.5305346884666372,
-      "eval_e3_equal": 0.8043305346884666,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.037384003535130356,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.993813521873619,
-      "eval_e3_heavy_atoms_difference": 0.3033141847105612,
-      "eval_e3_heavy_atoms_difference_norm": 0.0031366882148514846,
+      "eval_e3_has_attachment_point(s)": 0.9461776403004861,
+      "eval_e3_heavy_atoms_difference": 1.7942554131683606,
+      "eval_e3_heavy_atoms_difference_norm": 0.009207668134979698,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.993813521873619,
-      "eval_has_all_attachment_points": 0.989041095890411,
-      "eval_has_three_substructures": 0.9995581087052585,
-      "eval_heavy_atoms_difference": 3.9929297392841363,
-      "eval_heavy_atoms_difference_norm": 0.05206019398139567,
-      "eval_linker_equal": 0.762439239946973,
+      "eval_e3_valid": 0.9461776403004861,
+      "eval_has_all_attachment_points": 0.6704374723817941,
+      "eval_has_three_substructures": 0.9482103402562969,
+      "eval_heavy_atoms_difference": 64.90022094564738,
+      "eval_heavy_atoms_difference_norm": 0.8422759977934355,
+      "eval_linker_equal": 0.0007070260715863897,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9987627043747238,
-      "eval_linker_heavy_atoms_difference": 0.31524524966858153,
-      "eval_linker_heavy_atoms_difference_norm": -0.003168599779403479,
+      "eval_linker_has_attachment_point(s)": 0.9129474149359258,
+      "eval_linker_heavy_atoms_difference": 9.08882015024304,
+      "eval_linker_heavy_atoms_difference_norm": 0.45784852428455797,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9987627043747238,
-      "eval_loss": 0.2911645174026489,
-      "eval_num_fragments": 3.000265134776845,
-      "eval_poi_equal": 0.7620857269111798,
+      "eval_linker_valid": 0.9129474149359258,
+      "eval_loss": 0.7413397431373596,
+      "eval_num_fragments": 3.0321696862571805,
+      "eval_poi_equal": 8.837825894829872e-05,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9606716747680071,
-      "eval_poi_heavy_atoms_difference": 1.122227132125497,
-      "eval_poi_heavy_atoms_difference_norm": 0.0297152624658196,
+      "eval_poi_has_attachment_point(s)": 0.3013698630136986,
+      "eval_poi_heavy_atoms_difference": 26.02792752982766,
+      "eval_poi_heavy_atoms_difference_norm": 0.7980199520836797,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9606716747680071,
-      "eval_reassembly": 0.5367211665930182,
-      "eval_reassembly_nostereo": 0.5691559876270438,
-      "eval_runtime": 1513.244,
-      "eval_samples_per_second": 7.477,
-      "eval_steps_per_second": 0.117,
+      "eval_poi_valid": 0.3013698630136986,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 1500.6368,
+      "eval_samples_per_second": 7.54,
+      "eval_steps_per_second": 0.118,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9542200618647813,
+      "eval_valid": 0.2844896155545736,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.129548579454422,
-      "learning_rate": 2.397833750035968e-06,
-      "loss": 0.0057,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.1010139063000679,
-      "learning_rate": 3.897396719490167e-05,
-      "loss": 0.0059,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.18191708624362946,
-      "learning_rate": 8.777397133700313e-05,
-      "loss": 0.0102,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.13861054182052612,
-      "learning_rate": 0.00010283375410246977,
-      "loss": 0.0131,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.0767102986574173,
-      "learning_rate": 6.996870108781198e-05,
-      "loss": 0.0077,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.524259832081308,
-      "eval_e3_equal": 0.8037118868758285,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9929297392841361,
-      "eval_e3_heavy_atoms_difference": 0.36809544851966414,
-      "eval_e3_heavy_atoms_difference_norm": 0.00550609940775053,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9929297392841361,
-      "eval_has_all_attachment_points": 0.9876270437472382,
-      "eval_has_three_substructures": 0.9990278391515687,
-      "eval_heavy_atoms_difference": 4.941758727353071,
-      "eval_heavy_atoms_difference_norm": 0.06409348211177546,
-      "eval_linker_equal": 0.7583738400353514,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.996464869642068,
-      "eval_linker_heavy_atoms_difference": 0.3853292090145824,
-      "eval_linker_heavy_atoms_difference_norm": 0.0021846078803239542,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.996464869642068,
-      "eval_loss": 0.29665157198905945,
-      "eval_num_fragments": 2.999381352187362,
-      "eval_poi_equal": 0.759169244365886,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9536897923110915,
-      "eval_poi_heavy_atoms_difference": 1.4947414935925762,
-      "eval_poi_heavy_atoms_difference_norm": 0.04112715322890967,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9536897923110915,
-      "eval_reassembly": 0.5323906319045515,
-      "eval_reassembly_nostereo": 0.5604949182501104,
-      "eval_runtime": 1577.3192,
-      "eval_samples_per_second": 7.174,
-      "eval_steps_per_second": 0.112,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9458241272646929,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.07852062582969666,
-      "learning_rate": 2.0133980214319274e-05,
-      "loss": 0.0043,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.07622414827346802,
-      "learning_rate": 2.6826984838708546e-07,
-      "loss": 0.0031,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.08988434821367264,
-      "learning_rate": 2.908282537666773e-05,
-      "loss": 0.0031,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.14571209251880646,
-      "learning_rate": 7.943759001734402e-05,
-      "loss": 0.005,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.13105829060077667,
-      "learning_rate": 0.00010390406291286528,
-      "loss": 0.01,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5092355280600972,
-      "eval_e3_equal": 0.8045956694653116,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9835616438356164,
-      "eval_e3_heavy_atoms_difference": 0.6760936809544852,
-      "eval_e3_heavy_atoms_difference_norm": 0.021435481587991977,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9835616438356164,
-      "eval_has_all_attachment_points": 0.980468404772426,
-      "eval_has_three_substructures": 0.9977905435262925,
-      "eval_heavy_atoms_difference": 7.872912063632347,
-      "eval_heavy_atoms_difference_norm": 0.10373201914553183,
-      "eval_linker_equal": 0.743791427308882,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9939902783915157,
-      "eval_linker_heavy_atoms_difference": 0.6136986301369863,
-      "eval_linker_heavy_atoms_difference_norm": 0.0169426235349393,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9939902783915157,
-      "eval_loss": 0.30630069971084595,
-      "eval_num_fragments": 2.998320813079982,
-      "eval_poi_equal": 0.7374281926646045,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9276182059213434,
-      "eval_poi_heavy_atoms_difference": 2.429871851524525,
-      "eval_poi_heavy_atoms_difference_norm": 0.07454511298556593,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9276182059213434,
-      "eval_reassembly": 0.516570923552806,
-      "eval_reassembly_nostereo": 0.5413168360583297,
-      "eval_runtime": 2159.1811,
-      "eval_samples_per_second": 5.24,
-      "eval_steps_per_second": 0.082,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9110914714980115,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=11-learning_rate=1.0e-04-warmup_ratio=0.080-num_cycles=3.000",
+  "trial_name": "trial-number=12-learning_rate=2.2e-06-warmup_ratio=0.050-num_cycles=10.000",
   "trial_params": {
-    "learning_rate": 0.00010390406291286528,
-    "num_cycles": 3.0,
-    "warmup_ratio": 0.08
+    "learning_rate": 2.182809124107808e-06,
+    "num_cycles": 10.0,
+    "warmup_ratio": 0.05
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df253bfc107f1530386928af53b53aa5d613d5622ef461f94f7df0220f62cfd2
+oid sha256:5ab795fe306a3df71aedd0af760cbeb1cc92edd3774292b1be2346e1f0dde464
 size 7480