Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a49bb6d7eded9849b50d58961898463dba63258e6d769f5170315b0dbab4e76
+oid sha256:358d0a240c6a5817c69478888f79206e6c3e5e2ff96ea2b5f31b5da81c3b25b8
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:882125702a1b7fd1dd9a01efbf8982d98e09fdd9aeb12bd53a3eb67ac55eaa50
+oid sha256:74db241ad16485c10abf381c105389e9f33cafe0a7167f5347d4a280ed004b62
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47be9168f9422c2316ecd4af2ec720553ba0f002d2d4e1cc4fa003b7fb530aa8
+oid sha256:5360ad3c85351bd58e8cf8e520f4f343ea87be8760fabfa52521131f5c9ec4fd
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e3b218584cdbb2601f8e282cbe953ba899933915e1b964a6fa4dde0353919f4
+oid sha256:ca02b0e9c96a82fbe4ec920653c286bcb510fac2715c89468b5d0f8ed38fbe31
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.458859920459567,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=9-learning_rate=1.3e-05-warmup_ratio=0.050-min_lr=0.000-factor=0.890/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5155987627043748,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=10-learning_rate=5.0e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.970/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.0882785320281982,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 1.6004,
+      "grad_norm": 0.6662843823432922,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.953,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.7874086499214172,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.5219,
+      "grad_norm": 0.4416065216064453,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.1658,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.7524672150611877,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.3065,
+      "grad_norm": 0.31083112955093384,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.065,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.5783708691596985,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.2066,
+      "grad_norm": 0.21939702332019806,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0354,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.669140100479126,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.151,
+      "grad_norm": 0.2217538058757782,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0227,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.4841144382953644,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.1148,
+      "grad_norm": 0.16693812608718872,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0162,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.34032905101776123,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0882,
+      "grad_norm": 0.19489353895187378,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0121,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.32155144214630127,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0698,
+      "grad_norm": 0.11393198370933533,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0099,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.38137614727020264,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.056,
+      "grad_norm": 0.18439996242523193,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0083,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.39012080430984497,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0469,
+      "grad_norm": 0.12967944145202637,
+      "learning_rate": 5.0444751411035885e-05,
+      "loss": 0.0072,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.20972160848431287,
-      "eval_e3_equal": 0.7072028281042864,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9947856827220504,
-      "eval_e3_heavy_atoms_difference": 0.09491825011047282,
-      "eval_e3_heavy_atoms_difference_norm": -0.006944252067541642,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9947856827220504,
-      "eval_has_all_attachment_points": 0.9820592134334953,
-      "eval_has_three_substructures": 0.9984975695978789,
-      "eval_heavy_atoms_difference": 9.720901458241272,
-      "eval_heavy_atoms_difference_norm": 0.12639386628820087,
-      "eval_linker_equal": 0.33548387096774196,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9950508174988952,
-      "eval_linker_heavy_atoms_difference": 1.5590808661069377,
-      "eval_linker_heavy_atoms_difference_norm": 0.05308081409891352,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9950508174988952,
-      "eval_loss": 0.2932600677013397,
-      "eval_num_fragments": 2.999381352187362,
-      "eval_poi_equal": 0.636411842686699,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9011047282368537,
-      "eval_poi_heavy_atoms_difference": 3.2722934158197083,
-      "eval_poi_heavy_atoms_difference_norm": 0.1006642870975325,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9011047282368537,
-      "eval_reassembly": 0.2139637649138312,
-      "eval_reassembly_nostereo": 0.23932832523199293,
-      "eval_runtime": 1512.9908,
-      "eval_samples_per_second": 7.479,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8955368979231109,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.30534666776657104,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0392,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.2915492653846741,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0331,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.29193198680877686,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0282,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.23863498866558075,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0245,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.1951247751712799,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0211,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.38886433937251436,
-      "eval_e3_equal": 0.7711003093239063,
+      "eval_all_ligands_equal": 0.5155987627043748,
+      "eval_e3_equal": 0.7998232434821034,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9920459566946531,
-      "eval_e3_heavy_atoms_difference": 0.27741935483870966,
-      "eval_e3_heavy_atoms_difference_norm": 0.002726689436456998,
+      "eval_e3_has_attachment_point(s)": 0.980468404772426,
+      "eval_e3_heavy_atoms_difference": 0.5803800265134776,
+      "eval_e3_heavy_atoms_difference_norm": 0.013649380499657129,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9920459566946531,
-      "eval_has_all_attachment_points": 0.9852408307556341,
+      "eval_e3_valid": 0.980468404772426,
+      "eval_has_all_attachment_points": 0.9749889527176314,
       "eval_has_three_substructures": 0.9986743261157756,
-      "eval_heavy_atoms_difference": 8.22439239946973,
-      "eval_heavy_atoms_difference_norm": 0.10840505791645295,
-      "eval_linker_equal": 0.566593018117543,
+      "eval_heavy_atoms_difference": 8.000265134776845,
+      "eval_heavy_atoms_difference_norm": 0.10569745764971478,
+      "eval_linker_equal": 0.737251436146708,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9939019001325674,
-      "eval_linker_heavy_atoms_difference": 0.8546177640300486,
-      "eval_linker_heavy_atoms_difference_norm": 0.021245980777540405,
+      "eval_linker_has_attachment_point(s)": 0.9957578435704817,
+      "eval_linker_heavy_atoms_difference": 0.4416261599646487,
+      "eval_linker_heavy_atoms_difference_norm": 0.0036496355470818967,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9939019001325674,
-      "eval_loss": 0.29591697454452515,
-      "eval_num_fragments": 3.0000883782589485,
-      "eval_poi_equal": 0.6945647370746796,
+      "eval_linker_valid": 0.9957578435704817,
+      "eval_loss": 0.2973368465900421,
+      "eval_num_fragments": 2.9995581087052585,
+      "eval_poi_equal": 0.750861688024746,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9183384887317719,
-      "eval_poi_heavy_atoms_difference": 2.710384445426425,
-      "eval_poi_heavy_atoms_difference_norm": 0.08735808045935696,
+      "eval_poi_has_attachment_point(s)": 0.9195757843570481,
+      "eval_poi_heavy_atoms_difference": 2.296509058771542,
+      "eval_poi_heavy_atoms_difference_norm": 0.07321269145129682,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9183384887317719,
-      "eval_reassembly": 0.3954043305346885,
-      "eval_reassembly_nostereo": 0.4201502430402121,
-      "eval_runtime": 1458.0239,
-      "eval_samples_per_second": 7.761,
-      "eval_steps_per_second": 0.121,
+      "eval_poi_valid": 0.9195757843570481,
+      "eval_reassembly": 0.5221387538665488,
+      "eval_reassembly_nostereo": 0.5453822359699514,
+      "eval_runtime": 1650.2516,
+      "eval_samples_per_second": 6.857,
+      "eval_steps_per_second": 0.107,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9083517454706143,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.22444292902946472,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0189,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.19138626754283905,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.017,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.20459671318531036,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0149,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.17373104393482208,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0133,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.21692904829978943,
-      "learning_rate": 1.287188117510188e-05,
-      "loss": 0.0121,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.458859920459567,
-      "eval_e3_equal": 0.7884224480777728,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9934600088378259,
-      "eval_e3_heavy_atoms_difference": 0.37384003535130356,
-      "eval_e3_heavy_atoms_difference_norm": 0.006127300870536556,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9934600088378259,
-      "eval_has_all_attachment_points": 0.984622182942996,
-      "eval_has_three_substructures": 0.9984975695978789,
-      "eval_heavy_atoms_difference": 8.862660185594343,
-      "eval_heavy_atoms_difference_norm": 0.11505861502510897,
-      "eval_linker_equal": 0.6694653115333628,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9954043305346885,
-      "eval_linker_heavy_atoms_difference": 0.7417587273530711,
-      "eval_linker_heavy_atoms_difference_norm": 0.024397146955196757,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9954043305346885,
-      "eval_loss": 0.3033338785171509,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.7105612019443217,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9089703932832524,
-      "eval_poi_heavy_atoms_difference": 3.2495802032699954,
-      "eval_poi_heavy_atoms_difference_norm": 0.1031486423872984,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9089703932832524,
-      "eval_reassembly": 0.4669907202828104,
-      "eval_reassembly_nostereo": 0.4898806893504198,
-      "eval_runtime": 1583.3986,
-      "eval_samples_per_second": 7.146,
-      "eval_steps_per_second": 0.112,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.902960671674768,
-      "step": 10000
+      "eval_valid": 0.9002209456473708,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=9-learning_rate=1.3e-05-warmup_ratio=0.050-min_lr=0.000-factor=0.890",
+  "trial_name": "trial-number=10-learning_rate=5.0e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.970",
   "trial_params": {
-    "factor": 0.89,
-    "learning_rate": 1.287188117510188e-05,
-    "min_lr": 7.953202892241938e-10,
-    "warmup_ratio": 0.05
+    "factor": 0.9700000000000001,
+    "learning_rate": 5.0444751411035885e-05,
+    "min_lr": 1.3574361727870806e-09,
+    "warmup_ratio": 0.03
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f14039046a82de020163ad9a293b54d8c44c1c489589672206495ade799b622
+oid sha256:3feac406409e75e1cafe520313ae63f59c2f1993f9fd4435a951c83cd1936655
 size 7480