Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79808196a5a84d0cb2f05ba767a529b9d39d5218136e4c294d33e3e46c70ef18
+oid sha256:3ebcf628947539344ceabf7ebc899ef8df02dffe421cd545be0a587861b677d3
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff8812887e89c947496d8ab8cd71b420a24be6b82e488713e886e94f7913743f
+oid sha256:0dcbc0571930bcfbec38e9b3ad3f006283f8ce70e796ba399093f5d1028d150e
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74aad9d5b17d876be9e4d6b6f3353a4b59909eb061d599a7c44d6c2e03c798cf
+oid sha256:67843930f7b36709f116e7aef57b5c4339f568f2de52a675685f06f1fd05ec66
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07d8f542b00c07a19198f5221b28a0ac02536cfb19e79fe9ff0a366dc086c68c
+oid sha256:f0e71261348e9b0dc59a24363376590c3d78e50ba5d84bd7d40a5576639924da
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5381352187361909,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=17-learning_rate=2.8e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.680/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5203711886875828,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=18-learning_rate=6.4e-05-warmup_ratio=0.050-min_lr=0.000-factor=0.240/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.829456627368927,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 1.1848,
+      "grad_norm": 0.6651552319526672,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.8862,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.6500025391578674,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.2962,
+      "grad_norm": 0.36618563532829285,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.1584,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.47073793411254883,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.1468,
+      "grad_norm": 0.24036307632923126,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0632,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.329457551240921,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0878,
+      "grad_norm": 0.20140977203845978,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.036,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.3172358572483063,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0581,
+      "grad_norm": 0.20425930619239807,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0238,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.2527906894683838,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0424,
+      "grad_norm": 0.17172357439994812,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0176,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.20880313217639923,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0323,
+      "grad_norm": 0.1477121263742447,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0136,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.22946058213710785,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0257,
+      "grad_norm": 0.14009247720241547,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0117,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.22088561952114105,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0207,
+      "grad_norm": 0.15225987136363983,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0101,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.22565944492816925,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0175,
+      "grad_norm": 0.10719581693410873,
+      "learning_rate": 6.395962809058983e-05,
+      "loss": 0.0091,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.4470172337604949,
-      "eval_e3_equal": 0.7797613787008396,
+      "eval_all_ligands_equal": 0.5203711886875828,
+      "eval_e3_equal": 0.7986743261157755,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9953159522757402,
-      "eval_e3_heavy_atoms_difference": 0.2501988510826337,
-      "eval_e3_heavy_atoms_difference_norm": 0.0008378467337176671,
+      "eval_e3_has_attachment_point(s)": 0.9968183826778613,
+      "eval_e3_heavy_atoms_difference": 0.2883782589482987,
+      "eval_e3_heavy_atoms_difference_norm": 0.0026646216057636496,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9953159522757402,
-      "eval_has_all_attachment_points": 0.9885992045956694,
-      "eval_has_three_substructures": 0.9992929739284137,
-      "eval_heavy_atoms_difference": 5.620680512593902,
-      "eval_heavy_atoms_difference_norm": 0.07341529464537826,
-      "eval_linker_equal": 0.6448961555457358,
+      "eval_e3_valid": 0.9968183826778613,
+      "eval_has_all_attachment_points": 0.9934600088378259,
+      "eval_has_three_substructures": 0.9997348652231551,
+      "eval_heavy_atoms_difference": 3.5762262483429077,
+      "eval_heavy_atoms_difference_norm": 0.04541662513362855,
+      "eval_linker_equal": 0.7444984533804684,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9975254087494476,
-      "eval_linker_heavy_atoms_difference": 0.8009721608484313,
-      "eval_linker_heavy_atoms_difference_norm": 0.02116394604640135,
+      "eval_linker_has_attachment_point(s)": 0.9977905435262925,
+      "eval_linker_heavy_atoms_difference": 0.5421122403888643,
+      "eval_linker_heavy_atoms_difference_norm": 0.011771790126169518,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9975254087494476,
-      "eval_loss": 0.2864011228084564,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.7278833406981883,
+      "eval_linker_valid": 0.9977905435262925,
+      "eval_loss": 0.30581870675086975,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.7498895271763146,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9450287229341582,
-      "eval_poi_heavy_atoms_difference": 1.7335395492708794,
-      "eval_poi_heavy_atoms_difference_norm": 0.051064183804263204,
+      "eval_poi_has_attachment_point(s)": 0.9690676093680954,
+      "eval_poi_heavy_atoms_difference": 0.9461776403004861,
+      "eval_poi_heavy_atoms_difference_norm": 0.020891734527612723,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9450287229341582,
-      "eval_reassembly": 0.45417587273530713,
-      "eval_reassembly_nostereo": 0.4824569155987627,
-      "eval_runtime": 1504.245,
-      "eval_samples_per_second": 7.522,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9395492708793637,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.21528206765651703,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0151,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.14188343286514282,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0133,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.13938453793525696,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0116,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.14794708788394928,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0102,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.17807906866073608,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0092,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5155987627043748,
-      "eval_e3_equal": 0.7997348652231551,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9950508174988952,
-      "eval_e3_heavy_atoms_difference": 0.2891736632788334,
-      "eval_e3_heavy_atoms_difference_norm": 0.0028955297689230454,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9950508174988952,
-      "eval_has_all_attachment_points": 0.9878038002651348,
-      "eval_has_three_substructures": 0.9990278391515687,
-      "eval_heavy_atoms_difference": 5.2259832081308,
-      "eval_heavy_atoms_difference_norm": 0.06936488082967972,
-      "eval_linker_equal": 0.7362792752982766,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9962881131241714,
-      "eval_linker_heavy_atoms_difference": 0.46539991162174105,
-      "eval_linker_heavy_atoms_difference_norm": 0.007389768929469861,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9962881131241714,
-      "eval_loss": 0.3027403950691223,
-      "eval_num_fragments": 3.0004418912947415,
-      "eval_poi_equal": 0.7507733097657976,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.947326557666814,
-      "eval_poi_heavy_atoms_difference": 1.571011931064958,
-      "eval_poi_heavy_atoms_difference_norm": 0.04874062832136298,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.947326557666814,
-      "eval_reassembly": 0.5235528060097216,
-      "eval_reassembly_nostereo": 0.5519222271321255,
-      "eval_runtime": 1514.2596,
-      "eval_samples_per_second": 7.472,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9410517012814847,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.13021309673786163,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0083,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.15480028092861176,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0078,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.13444596529006958,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0069,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.1448230892419815,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0065,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.09779265522956848,
-      "learning_rate": 2.8331451115475792e-05,
-      "loss": 0.0061,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5381352187361909,
-      "eval_e3_equal": 0.8015908086610694,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9930181175430844,
-      "eval_e3_heavy_atoms_difference": 0.3586389748121962,
-      "eval_e3_heavy_atoms_difference_norm": 0.004046618972503898,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9930181175430844,
-      "eval_has_all_attachment_points": 0.9855943437914273,
-      "eval_has_three_substructures": 0.9994697304463102,
-      "eval_heavy_atoms_difference": 4.982677861246134,
-      "eval_heavy_atoms_difference_norm": 0.06595580402806239,
-      "eval_linker_equal": 0.7677419354838709,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9951391957578436,
-      "eval_linker_heavy_atoms_difference": 0.29235528060097216,
-      "eval_linker_heavy_atoms_difference_norm": -0.001514220633297117,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9951391957578436,
-      "eval_loss": 0.3127844035625458,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.7611135660627486,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9520106053910738,
-      "eval_poi_heavy_atoms_difference": 1.4593901900132566,
-      "eval_poi_heavy_atoms_difference_norm": 0.04606472460308478,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9520106053910738,
-      "eval_reassembly": 0.5465311533362793,
-      "eval_reassembly_nostereo": 0.5773751657092355,
-      "eval_runtime": 1527.8773,
-      "eval_samples_per_second": 7.406,
+      "eval_poi_valid": 0.9690676093680954,
+      "eval_reassembly": 0.5271763146266019,
+      "eval_reassembly_nostereo": 0.5572249226690235,
+      "eval_runtime": 1528.5096,
+      "eval_samples_per_second": 7.403,
       "eval_steps_per_second": 0.116,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9422889969067609,
-      "step": 10000
+      "eval_valid": 0.9646486964206805,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=17-learning_rate=2.8e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.680",
+  "trial_name": "trial-number=18-learning_rate=6.4e-05-warmup_ratio=0.050-min_lr=0.000-factor=0.240",
   "trial_params": {
-    "factor": 0.6799999999999999,
-    "learning_rate": 2.8331451115475792e-05,
-    "min_lr": 7.147010870778695e-11,
-    "warmup_ratio": 0.060000000000000005
+    "factor": 0.24000000000000002,
+    "learning_rate": 6.395962809058983e-05,
+    "min_lr": 2.960963543629417e-11,
+    "warmup_ratio": 0.05
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fcc15b3e8d0f3d92cb620d02832c5928bcf97a45dce5fd03bf77c673c58eafa
+oid sha256:a4caf8b92e41afe1413a9109a3dd0d39d9313ce66cb8219e2d7a9d77a7c66cd7
 size 7544