Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55a9106cdb8c62c8cb6c6cf2ccc75d943d297cbb37eb48f2888db56021a9cf39
+oid sha256:56460d2d87bb45f60b8e314d21bda8850c8dd3b06d7d5e847c4593ef325ddca1
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0446a49c3bb5dd0c3ac104ec08d4544d44bed7f66667e91c392afa1b706980ca
+oid sha256:c2da71d54eab307cb2da248a83b0679ef72c5fefd443a67aaa5582cbda202adc
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afe240b3e703829fba42c3f2a3fa53ecc7ce4171cc7b64de5bd39cdada6c5594
+oid sha256:9cf2bdb4ab13de64cc25d58bdf8f6d30a26a8d22776dad2b72bcaac6ee4429f8
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f5f0f04e739bbfb1fc5cae716a665897a5f3c6c64a428de8cb172be54e15298
+oid sha256:197841552fe7ec3bc9b1e219bc5bda90e3cfc822804f93b84cf07d60e62c21a2
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=7-learning_rate=1.8e-04-warmup_ratio=0.100-min_lr=0.000-factor=0.710/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.158285461776403,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=8-learning_rate=1.3e-05-warmup_ratio=0.010-min_lr=0.000-factor=0.220/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.49629172682762146,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.9567,
+      "grad_norm": 0.9919585585594177,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 1.6025,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.34096312522888184,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1992,
+      "grad_norm": 0.9148474931716919,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.536,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.24350646138191223,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1742,
+      "grad_norm": 0.7302022576332092,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.3244,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.2776834964752197,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1627,
+      "grad_norm": 0.6461985111236572,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.2247,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.2741040885448456,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1553,
+      "grad_norm": 0.5478264689445496,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.1687,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.3086952269077301,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1516,
+      "grad_norm": 0.46242278814315796,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.1336,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.38999202847480774,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1474,
+      "grad_norm": 0.36126068234443665,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.1064,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.2551262378692627,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1454,
+      "grad_norm": 0.3201972246170044,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.0862,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.2856215238571167,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1424,
+      "grad_norm": 0.35753563046455383,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.0704,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.5204154253005981,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1431,
+      "grad_norm": 0.3595835566520691,
+      "learning_rate": 1.2789275019323142e-05,
+      "loss": 0.0591,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0346442775077331,
+      "eval_all_ligands_equal": 0.158285461776403,
+      "eval_e3_equal": 0.6709677419354839,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 1.5730446310207689,
-      "eval_e3_heavy_atoms_difference_norm": -0.0005120283470902502,
+      "eval_e3_has_attachment_point(s)": 0.9984091913389306,
+      "eval_e3_heavy_atoms_difference": -0.1937251436146708,
+      "eval_e3_heavy_atoms_difference_norm": -0.022125648951058074,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 3.9468846663720725,
-      "eval_heavy_atoms_difference_norm": 0.023292187761076128,
-      "eval_linker_equal": 0.0023862129916040653,
+      "eval_e3_valid": 0.9984091913389306,
+      "eval_has_all_attachment_points": 0.9885992045956694,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 6.95793194874061,
+      "eval_heavy_atoms_difference_norm": 0.08973210339619288,
+      "eval_linker_equal": 0.268935041979673,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": 0.4876712328767123,
-      "eval_linker_heavy_atoms_difference_norm": -0.19550334215272724,
+      "eval_linker_has_attachment_point(s)": 0.9971718957136544,
+      "eval_linker_heavy_atoms_difference": 1.9495360141405214,
+      "eval_linker_heavy_atoms_difference_norm": 0.07550810263300518,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.4921937882900238,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.003535130357931949,
+      "eval_linker_valid": 0.9971718957136544,
+      "eval_loss": 0.2988537549972534,
+      "eval_num_fragments": 2.999734865223155,
+      "eval_poi_equal": 0.6062748563853292,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": 1.8861688024745913,
-      "eval_poi_heavy_atoms_difference_norm": -0.07049150153739635,
+      "eval_poi_has_attachment_point(s)": 0.9368095448519664,
+      "eval_poi_heavy_atoms_difference": 2.0343791427308884,
+      "eval_poi_heavy_atoms_difference_norm": 0.06010484602679077,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1323.3177,
-      "eval_samples_per_second": 8.55,
-      "eval_steps_per_second": 0.134,
+      "eval_poi_valid": 0.9368095448519664,
+      "eval_reassembly": 0.16217410517012815,
+      "eval_reassembly_nostereo": 0.18382677861246133,
+      "eval_runtime": 1527.9721,
+      "eval_samples_per_second": 7.405,
+      "eval_steps_per_second": 0.116,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
+      "eval_valid": 0.9341581970835174,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.23685993254184723,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.147,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.20809854567050934,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.142,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.2273290902376175,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1446,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.3077870309352875,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1399,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 5.584017276763916,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1386,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.03543968183826779,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 1.34856385329209,
-      "eval_e3_heavy_atoms_difference_norm": -0.0034216972448036422,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -0.8091913389306231,
-      "eval_heavy_atoms_difference_norm": -0.03921231630974343,
-      "eval_linker_equal": 0.0014140521431727795,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -0.5910737958462219,
-      "eval_linker_heavy_atoms_difference_norm": -0.2813864737304455,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5082502365112305,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0021210782147591694,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9996464869642068,
-      "eval_poi_heavy_atoms_difference": -1.581617322138754,
-      "eval_poi_heavy_atoms_difference_norm": -0.19857852436254525,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9996464869642068,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1485.1844,
-      "eval_samples_per_second": 7.619,
-      "eval_steps_per_second": 0.119,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9996464869642068,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2829979956150055,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1509,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.4774325489997864,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1483,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.4473247826099396,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1487,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.5797591209411621,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1435,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.8456087708473206,
-      "learning_rate": 0.00017553249516694252,
-      "loss": 0.1411,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.02483429076447194,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 2.2069818824569154,
-      "eval_e3_heavy_atoms_difference_norm": 0.028085955508926997,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 1.5678303137428193,
-      "eval_heavy_atoms_difference_norm": -0.009803369302753544,
-      "eval_linker_equal": 0.001855943437914273,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -0.32832523199292973,
-      "eval_linker_heavy_atoms_difference_norm": -0.25687187670805983,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.558329701423645,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.001855943437914273,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -0.3108263367211666,
-      "eval_poi_heavy_atoms_difference_norm": -0.15080735877161097,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1427.9008,
-      "eval_samples_per_second": 7.924,
-      "eval_steps_per_second": 0.124,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=7-learning_rate=1.8e-04-warmup_ratio=0.100-min_lr=0.000-factor=0.710",
+  "trial_name": "trial-number=8-learning_rate=1.3e-05-warmup_ratio=0.010-min_lr=0.000-factor=0.220",
   "trial_params": {
-    "factor": 0.71,
-    "learning_rate": 0.00017553249516694252,
-    "min_lr": 1.836881026598859e-11,
-    "warmup_ratio": 0.09999999999999999
+    "factor": 0.22,
+    "learning_rate": 1.2789275019323142e-05,
+    "min_lr": 3.966711717977572e-11,
+    "warmup_ratio": 0.01
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e535304a50d6c7bf62265d8313d99249f4fa14351f55f318cc66b31cfb03d0c
+oid sha256:7482535826fb3d396cd1599682322d70f83ade3cf9e3ac8479e85eef0257ec8a
 size 7544