Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2aa779d02fb1ca052d47d53b3fb468aededee3ecddf0c215c8e1f35d0668cbce
+oid sha256:76ef86f9310260be3a7d83639ac2adb1847e51997c1962cc2287d2846e4acc20
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce8223b0e87e3a2f649ca96b1b0f349dc2718f40c82bd4ca6299eb32baa11b6c
+oid sha256:730d61ea4491cdfa52cf37ff537d4dddbf445b4821beb1b645e49b4019d170c6
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0905785dc927bee6dc52f7ee2976eb731655682470d0dd9f62c15bfd74f2515e
+oid sha256:9db9fd9e046de6d30a56058133cf9d1aa9b3478eabf403973868620c7435e816
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a152ffd4eaf1c6ba79374d58ed69d0726d949ea781eae0e2247e20698d7774d5
+oid sha256:6bf59fb2d8b8cce1bcc3d0343091997127b852fff1ad02bb5e7a49c66aa5e07b
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=19-learning_rate=1.8e-04-warmup_ratio=0.100-min_lr=0.000-factor=0.930/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 8.837825894829872e-05,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=20-learning_rate=2.5e-06-warmup_ratio=0.010-min_lr=0.000-factor=0.440/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.4460906684398651,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 1.0142,
+      "grad_norm": 1.696872353553772,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 3.0863,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.3451838791370392,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.2001,
+      "grad_norm": 1.3597171306610107,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 1.2889,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.24968214333057404,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1756,
+      "grad_norm": 1.2895184755325317,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.9841,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.2555454969406128,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.168,
+      "grad_norm": 1.3098210096359253,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.8021,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.28854918479919434,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1633,
+      "grad_norm": 1.335889220237732,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.6762,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.2615298628807068,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1605,
+      "grad_norm": 1.3646761178970337,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.5867,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.25099700689315796,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1589,
+      "grad_norm": 1.076676607131958,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.5147,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.2410382330417633,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1569,
+      "grad_norm": 1.3067148923873901,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.4589,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.2645629644393921,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1554,
+      "grad_norm": 1.1095086336135864,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.4094,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.2651103734970093,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1543,
+      "grad_norm": 0.9933345317840576,
+      "learning_rate": 2.457364779031449e-06,
+      "loss": 0.3711,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.01449403446752099,
+      "eval_all_ligands_equal": 8.837825894829872e-05,
+      "eval_e3_equal": 0.12505523641184269,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 8.672381794078657,
-      "eval_e3_heavy_atoms_difference_norm": 0.2408192904697644,
+      "eval_e3_has_attachment_point(s)": 0.9908086610693769,
+      "eval_e3_heavy_atoms_difference": -1.127529827662395,
+      "eval_e3_heavy_atoms_difference_norm": -0.07319330834111164,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 8.089173663278833,
-      "eval_heavy_atoms_difference_norm": 0.07232688019696262,
-      "eval_linker_equal": 0.0016791869200176756,
+      "eval_e3_valid": 0.9908086610693769,
+      "eval_has_all_attachment_points": 0.8904993371630578,
+      "eval_has_three_substructures": 0.9921343349536014,
+      "eval_heavy_atoms_difference": 39.393725143614674,
+      "eval_heavy_atoms_difference_norm": 0.5032680394400146,
+      "eval_linker_equal": 0.0027397260273972603,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -1.4842244807777287,
-      "eval_linker_heavy_atoms_difference_norm": -0.34404268961334245,
+      "eval_linker_has_attachment_point(s)": 0.9836500220945648,
+      "eval_linker_heavy_atoms_difference": 7.094476358815731,
+      "eval_linker_heavy_atoms_difference_norm": 0.32585817993662086,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5216243863105774,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0016791869200176756,
+      "eval_linker_valid": 0.9836500220945648,
+      "eval_loss": 0.4786160886287689,
+      "eval_num_fragments": 2.99584622182943,
+      "eval_poi_equal": 0.043216968625718076,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": 0.9010163499779055,
-      "eval_poi_heavy_atoms_difference_norm": -0.11167547976496513,
+      "eval_poi_has_attachment_point(s)": 0.5878038002651348,
+      "eval_poi_heavy_atoms_difference": 16.118161732213874,
+      "eval_poi_heavy_atoms_difference_norm": 0.4666094747986684,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1193.9687,
-      "eval_samples_per_second": 9.477,
-      "eval_steps_per_second": 0.148,
+      "eval_poi_valid": 0.5878038002651348,
+      "eval_reassembly": 8.837825894829872e-05,
+      "eval_reassembly_nostereo": 8.837825894829872e-05,
+      "eval_runtime": 1462.6047,
+      "eval_samples_per_second": 7.736,
+      "eval_steps_per_second": 0.121,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
+      "eval_valid": 0.5842686699072028,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.27524861693382263,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1543,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.2908000648021698,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1523,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.27331531047821045,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1527,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.3615107834339142,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1514,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.29473376274108887,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1503,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.006716747680070702,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -6.550419796730004,
-      "eval_e3_heavy_atoms_difference_norm": -0.3544036689079038,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -20.141140079540435,
-      "eval_heavy_atoms_difference_norm": -0.3160320293892404,
-      "eval_linker_equal": 0.001590808661069377,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -2.135218736190897,
-      "eval_linker_heavy_atoms_difference_norm": -0.3955390343159519,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5303946137428284,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.001237295625276182,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -11.455501546619532,
-      "eval_poi_heavy_atoms_difference_norm": -0.565083423857184,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1564.7984,
-      "eval_samples_per_second": 7.231,
-      "eval_steps_per_second": 0.113,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.28325650095939636,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1495,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.3338431119918823,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1498,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.2506033182144165,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1531,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.3380557894706726,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1481,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.3548344075679779,
-      "learning_rate": 0.0001809696468670689,
-      "loss": 0.1476,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.005479452054794521,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -5.9238179407865665,
-      "eval_e3_heavy_atoms_difference_norm": -0.33000060419236454,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -9.478391515687141,
-      "eval_heavy_atoms_difference_norm": -0.16956106665343912,
-      "eval_linker_equal": 0.0011489173663278833,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": 0.8265134776844896,
-      "eval_linker_heavy_atoms_difference_norm": -0.16816308209299535,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5292195081710815,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0016791869200176756,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -4.381087052585064,
-      "eval_poi_heavy_atoms_difference_norm": -0.3046896586238993,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1608.355,
-      "eval_samples_per_second": 7.035,
-      "eval_steps_per_second": 0.11,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=19-learning_rate=1.8e-04-warmup_ratio=0.100-min_lr=0.000-factor=0.930",
+  "trial_name": "trial-number=20-learning_rate=2.5e-06-warmup_ratio=0.010-min_lr=0.000-factor=0.440",
   "trial_params": {
-    "factor": 0.93,
-    "learning_rate": 0.0001809696468670689,
-    "min_lr": 6.698351404357494e-10,
-    "warmup_ratio": 0.09999999999999999
+    "factor": 0.44000000000000006,
+    "learning_rate": 2.457364779031449e-06,
+    "min_lr": 1.6526441786125923e-12,
+    "warmup_ratio": 0.01
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37da6915186756900398f616b2334b9e5d55eb717e2d6108a5b95d1a6354e0f2
+oid sha256:71e90c3f230c85c3d05ed6919edda99c6e6504d973093c1f533e8012d44b6c0a
 size 7544