Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04dba00b6ac37f75f60552292f27aca6bad0b8dbb797a3729f15e4bb501430b1
+oid sha256:742360e8e083aa0ab2b8402c2742e80b6ad956f4987295b31a399ebedf292ad8
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:327c40e1eb4791180dd79695994f486aa000da083c7bb69e4647fa96eb864947
+oid sha256:283be59940b255cf15c5f400499dafc24d0050635cda315b33db732426c34c9b
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0949dee60d1643b5a7647dee6c8a05f8064ae2abcd43474f23666f0a9bef5416
+oid sha256:3aec273066a34c58ea565c0fe9241cf1529e2fff36103dc5c951ca4680600024
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:702a8ac1fa8a0499d41d32ca86b79a9b3b30f418c7f846489ebfbabf07221600
+oid sha256:dfa920d8527877e938cb3d3c463b964a9ba038554e784c6aba1cdb0339bb4d72
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.06389748121961997,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=13-learning_rate=9.3e-06-warmup_ratio=0.060-min_lr=0.000-factor=0.910/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.37693327441449404,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=13-learning_rate=9.3e-06-warmup_ratio=0.060-min_lr=0.000-factor=0.910/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8827220503756076,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.42155012488365173,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0716,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.3158087432384491,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0611,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.4184732735157013,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0532,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.32678478956222534,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.047,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.2851454019546509,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0404,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.23915156871409632,
+      "eval_e3_equal": 0.7059655324790102,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9959346000883783,
+      "eval_e3_heavy_atoms_difference": 0.07706584180291648,
+      "eval_e3_heavy_atoms_difference_norm": -0.008165564237987666,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9959346000883783,
+      "eval_has_all_attachment_points": 0.9825894829871852,
+      "eval_has_three_substructures": 0.9984975695978789,
+      "eval_heavy_atoms_difference": 8.136986301369863,
+      "eval_heavy_atoms_difference_norm": 0.10794979436441494,
+      "eval_linker_equal": 0.37454706142288996,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9955810870525851,
+      "eval_linker_heavy_atoms_difference": 1.2577993813521873,
+      "eval_linker_heavy_atoms_difference_norm": 0.043365232760132336,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9955810870525851,
+      "eval_loss": 0.2948099672794342,
+      "eval_num_fragments": 3.000618647812638,
+      "eval_poi_equal": 0.6457799381352187,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9172779496243924,
+      "eval_poi_heavy_atoms_difference": 2.6791869200176754,
+      "eval_poi_heavy_atoms_difference_norm": 0.08429235688620283,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9172779496243924,
+      "eval_reassembly": 0.24357048166151127,
+      "eval_reassembly_nostereo": 0.27105612019443215,
+      "eval_runtime": 1512.3816,
+      "eval_samples_per_second": 7.482,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9122403888643393,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.2971186637878418,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0357,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.28444504737854004,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0318,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.23130664229393005,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0287,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.23987413942813873,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0259,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.2695273458957672,
+      "learning_rate": 9.294928212675126e-06,
+      "loss": 0.0229,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.37693327441449404,
+      "eval_e3_equal": 0.7546619531595228,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9917808219178083,
+      "eval_e3_heavy_atoms_difference": 0.19602297834732657,
+      "eval_e3_heavy_atoms_difference_norm": -0.00033500808410158304,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9917808219178083,
+      "eval_has_all_attachment_points": 0.984622182942996,
+      "eval_has_three_substructures": 0.997613787008396,
+      "eval_heavy_atoms_difference": 7.414405656208572,
+      "eval_heavy_atoms_difference_norm": 0.09885276627689286,
+      "eval_linker_equal": 0.5522757401679187,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.992311091471498,
+      "eval_linker_heavy_atoms_difference": 0.5194874060980998,
+      "eval_linker_heavy_atoms_difference_norm": 0.000631865175113835,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.992311091471498,
+      "eval_loss": 0.29792699217796326,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.6930623066725585,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9171895713654441,
+      "eval_poi_heavy_atoms_difference": 2.443216968625718,
+      "eval_poi_heavy_atoms_difference_norm": 0.07682263793071213,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9171895713654441,
+      "eval_reassembly": 0.38320813079982324,
+      "eval_reassembly_nostereo": 0.41175430844012373,
+      "eval_runtime": 1477.3084,
+      "eval_samples_per_second": 7.659,
+      "eval_steps_per_second": 0.12,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9083517454706143,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=13-learning_rate=9.3e-06-warmup_ratio=0.060-min_lr=0.000-factor=0.910",
   "trial_params": {