Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13c3c5ab503aee696c8f8ee2693db3bc10e721df8e0ccaa6248bbd40e3e06ec5
+oid sha256:1eb5820eff8a87d2e55e43e91bc97234b2eb2c99dc6440cd4a796f86df809438
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bef7999393325229d960509011f84242cd933dc33c9f2bc3fe842542a45db5b0
+oid sha256:f74878d4a3fba218445bf8d27203512b0f279d24225959c99f1a28816bfb85cf
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fca754166078c1822bba5b68f8f68dc77082ffea1a4eb8fec2542f0bb1e6af0
+oid sha256:513ce340808a94186574d745db0d059bcbe90c4520fd9950974b8e01594f72e5
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fd1611e4ea9d20890ab10928d6c76fdd332d1eb1cce70fd9c7927e50946b909
+oid sha256:e5abb4045b7a90cdf6ddf50a8c38ea41405bd80989036a2b83584448ef66bb42
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 8.837825894829872e-05,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=4-learning_rate=2.6e-06-warmup_ratio=0.030-num_cycles=0.500/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=5-learning_rate=1.3e-06-warmup_ratio=0.080-num_cycles=4.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 3.9426004886627197,
-      "learning_rate": 1.84116374394636e-06,
-      "loss": 5.7738,
+      "grad_norm": 4.204223155975342,
+      "learning_rate": 9.653298476903908e-07,
+      "loss": 6.6322,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.4991477727890015,
-      "learning_rate": 2.567301241779092e-06,
-      "loss": 1.7738,
+      "grad_norm": 2.3499550819396973,
+      "learning_rate": 1.1383762270122183e-06,
+      "loss": 2.6514,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.3277864456176758,
-      "learning_rate": 2.5271309141899734e-06,
-      "loss": 1.1256,
+      "grad_norm": 1.8771620988845825,
+      "learning_rate": 2.973196899345272e-07,
+      "loss": 1.5869,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.3548855781555176,
-      "learning_rate": 2.4516727688213163e-06,
-      "loss": 0.8898,
+      "grad_norm": 2.109788179397583,
+      "learning_rate": 4.662220845232186e-08,
+      "loss": 1.4691,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.3984326124191284,
-      "learning_rate": 2.343073914710247e-06,
-      "loss": 0.7377,
+      "grad_norm": 1.5886791944503784,
+      "learning_rate": 7.783970331251584e-07,
+      "loss": 1.4377,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.4253393411636353,
-      "learning_rate": 2.204424456708191e-06,
-      "loss": 0.6356,
+      "grad_norm": 1.5623325109481812,
+      "learning_rate": 1.3480829841452437e-06,
+      "loss": 1.2951,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.1617372035980225,
-      "learning_rate": 2.0396695686968327e-06,
-      "loss": 0.5597,
+      "grad_norm": 1.3321806192398071,
+      "learning_rate": 8.646403560799816e-07,
+      "loss": 1.113,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.3310484886169434,
-      "learning_rate": 1.853497236409607e-06,
-      "loss": 0.5046,
+      "grad_norm": 1.3972088098526,
+      "learning_rate": 8.421659496257073e-08,
+      "loss": 1.0169,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.1124403476715088,
-      "learning_rate": 1.6512048640586581e-06,
-      "loss": 0.4586,
+      "grad_norm": 1.4411838054656982,
+      "learning_rate": 2.2746416198410507e-07,
+      "loss": 0.9966,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 1.103158712387085,
-      "learning_rate": 1.4385485403829063e-06,
-      "loss": 0.4262,
+      "grad_norm": 1.3339537382125854,
+      "learning_rate": 1.0703310690373619e-06,
+      "loss": 0.9726,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.09995581087052585,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9873619089703933,
-      "eval_e3_heavy_atoms_difference": -1.1366327883340699,
-      "eval_e3_heavy_atoms_difference_norm": -0.07616934597398835,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9873619089703933,
-      "eval_has_all_attachment_points": 0.845868316394167,
-      "eval_has_three_substructures": 0.9891294741493593,
-      "eval_heavy_atoms_difference": 46.22129916040654,
-      "eval_heavy_atoms_difference_norm": 0.5907942514452658,
-      "eval_linker_equal": 0.001855943437914273,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9816173221387539,
-      "eval_linker_heavy_atoms_difference": 7.361025187803801,
-      "eval_linker_heavy_atoms_difference_norm": 0.3407921741657837,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9816173221387539,
-      "eval_loss": 0.524074375629425,
-      "eval_num_fragments": 2.997437030490499,
-      "eval_poi_equal": 0.024038886433937253,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.5041979673000442,
-      "eval_poi_heavy_atoms_difference": 19.35581087052585,
-      "eval_poi_heavy_atoms_difference_norm": 0.5634695384472664,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.5041979673000442,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1401.3785,
-      "eval_samples_per_second": 8.074,
-      "eval_steps_per_second": 0.126,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.5001325673884225,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.7723095417022705,
-      "learning_rate": 1.2215792531469616e-06,
-      "loss": 0.3994,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.9387763142585754,
-      "learning_rate": 1.006470712493407e-06,
-      "loss": 0.3789,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.8577531576156616,
-      "learning_rate": 7.993436823151736e-07,
-      "loss": 0.363,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 1.0202293395996094,
-      "learning_rate": 6.060918181766256e-07,
-      "loss": 0.3512,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.6962732672691345,
-      "learning_rate": 4.32213967444154e-07,
-      "loss": 0.3424,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.12505523641184269,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9883340698188245,
-      "eval_e3_heavy_atoms_difference": -1.5878038002651347,
-      "eval_e3_heavy_atoms_difference_norm": -0.08971506122559236,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9883340698188245,
-      "eval_has_all_attachment_points": 0.8798055678303137,
-      "eval_has_three_substructures": 0.9896597437030491,
-      "eval_heavy_atoms_difference": 38.27697746354397,
-      "eval_heavy_atoms_difference_norm": 0.4870355739912452,
-      "eval_linker_equal": 0.002209456473707468,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9818824569155987,
-      "eval_linker_heavy_atoms_difference": 6.498011489173663,
-      "eval_linker_heavy_atoms_difference_norm": 0.28279271450702187,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9818824569155987,
-      "eval_loss": 0.4664870500564575,
-      "eval_num_fragments": 2.9939019001325673,
-      "eval_poi_equal": 0.05055236411842687,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.5932832523199293,
-      "eval_poi_heavy_atoms_difference": 15.902165267344234,
-      "eval_poi_heavy_atoms_difference_norm": 0.4558066183668017,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.5932832523199293,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1470.262,
-      "eval_samples_per_second": 7.696,
-      "eval_steps_per_second": 0.12,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.5893946089262041,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.8521112203598022,
-      "learning_rate": 2.8265770340941057e-07,
-      "loss": 0.3376,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.7935061454772949,
-      "learning_rate": 1.616785455328715e-07,
-      "loss": 0.3327,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.7593372464179993,
-      "learning_rate": 7.271887159776171e-08,
-      "loss": 0.3309,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.9376997947692871,
-      "learning_rate": 1.8309967244785057e-08,
-      "loss": 0.3295,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.7058411836624146,
-      "learning_rate": 0.0,
-      "loss": 0.3286,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
       "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.1254971277065842,
+      "eval_e3_equal": 0.02262483429076447,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9893946089262041,
-      "eval_e3_heavy_atoms_difference": -1.7589041095890412,
-      "eval_e3_heavy_atoms_difference_norm": -0.09673893136908718,
+      "eval_e3_has_attachment_point(s)": 0.7847989394608926,
+      "eval_e3_heavy_atoms_difference": 7.129032258064516,
+      "eval_e3_heavy_atoms_difference_norm": 0.19677256697299073,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9893946089262041,
-      "eval_has_all_attachment_points": 0.8814847547503314,
-      "eval_has_three_substructures": 0.9902783915156872,
-      "eval_heavy_atoms_difference": 36.80521431727795,
-      "eval_heavy_atoms_difference_norm": 0.4670994140058877,
-      "eval_linker_equal": 0.0030932390631904553,
+      "eval_e3_valid": 0.7847989394608926,
+      "eval_has_all_attachment_points": 0.4887317719840919,
+      "eval_has_three_substructures": 0.7915156871409633,
+      "eval_heavy_atoms_difference": 71.48528501988511,
+      "eval_heavy_atoms_difference_norm": 0.937024381393544,
+      "eval_linker_equal": 0.0004418912947414936,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.984003535130358,
-      "eval_linker_heavy_atoms_difference": 6.480070702607159,
-      "eval_linker_heavy_atoms_difference_norm": 0.2838460154041233,
+      "eval_linker_has_attachment_point(s)": 0.7076447193990278,
+      "eval_linker_heavy_atoms_difference": 11.91789659743703,
+      "eval_linker_heavy_atoms_difference_norm": 0.6676007790312372,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.984003535130358,
-      "eval_loss": 0.45746952295303345,
-      "eval_num_fragments": 2.995934600088378,
-      "eval_poi_equal": 0.0554131683605833,
+      "eval_linker_valid": 0.7076447193990278,
+      "eval_loss": 0.9779880046844482,
+      "eval_num_fragments": 3.163323022536456,
+      "eval_poi_equal": 0.0,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.610075121520106,
-      "eval_poi_heavy_atoms_difference": 15.067432611577551,
-      "eval_poi_heavy_atoms_difference_norm": 0.4293403966560753,
+      "eval_poi_has_attachment_point(s)": 0.1647370746796288,
+      "eval_poi_heavy_atoms_difference": 29.18179407865665,
+      "eval_poi_heavy_atoms_difference_norm": 0.9227533827119762,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.610075121520106,
+      "eval_poi_valid": 0.1647370746796288,
       "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1470.9165,
-      "eval_samples_per_second": 7.692,
-      "eval_steps_per_second": 0.12,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 2245.7621,
+      "eval_samples_per_second": 5.038,
+      "eval_steps_per_second": 0.079,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.6067167476800707,
-      "step": 10000
+      "eval_valid": 0.15377817057003976,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=4-learning_rate=2.6e-06-warmup_ratio=0.030-num_cycles=0.500",
+  "trial_name": "trial-number=5-learning_rate=1.3e-06-warmup_ratio=0.080-num_cycles=4.000",
   "trial_params": {
-    "learning_rate": 2.5739469140370113e-06,
-    "num_cycles": 0.5,
-    "warmup_ratio": 0.03
+    "learning_rate": 1.3495311270711663e-06,
+    "num_cycles": 4.0,
+    "warmup_ratio": 0.08
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d15479ab725266d90ed254e514f85d555faa041795946d208197ff040f56b6f
+oid sha256:063eba5a79ecf9f1043b5a071d5290a9003e89ebd5df2f1ede92f5651723aac3
 size 7480