Training in progress, step 100000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2da57805ae3d56e0392f41ff4c956ee56d01b9f4b97862863907a19d6e296bb2
+oid sha256:428e27d2fd4fa239aa4b31b7ba0c5dc9460e6289064d365c3089524789af6ab0
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b3c68ab591e0843618ff867c3358e9118ebffde4bc478e91736fd17cec98505
+oid sha256:8ef55722db971152f6745af575614a7ce45c6dc96133ee5e7911db97426b8d47
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6697b2eca1bae319dda07df8ffa6135644ca10cf76eaff3196d87074f84c9663
+oid sha256:189285e7807ec97b8899d04629066907a7db5ad0bdfa1b2dca374634a2f173e4
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebc7c5e53f81d6694acd5dc12ee279c4c3514f2c026c6f27c133cee0542adc4a
+oid sha256:32ef2d7e44c1881924f257e8f6478e73f99e51c274b68408ccff8e05b73cdc44
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.5964058073654391,
   "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-rand-smiles/checkpoint-95000",
-  "epoch": 9.370684553166305,
+  "epoch": 9.863878477017163,
   "eval_steps": 2500,
-  "global_step": 95000,
+  "global_step": 100000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1925,6 +1925,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9463526912181303,
       "step": 95000
+    },
+    {
+      "epoch": 9.420003945551391,
+      "grad_norm": 0.0025114284362643957,
+      "learning_rate": 2.529100918672256e-07,
+      "loss": 0.0002,
+      "step": 95500
+    },
+    {
+      "epoch": 9.469323337936476,
+      "grad_norm": 0.001364166266284883,
+      "learning_rate": 2.0019883840954324e-07,
+      "loss": 0.0002,
+      "step": 96000
+    },
+    {
+      "epoch": 9.518642730321563,
+      "grad_norm": 0.010334326885640621,
+      "learning_rate": 1.5361882088972267e-07,
+      "loss": 0.0002,
+      "step": 96500
+    },
+    {
+      "epoch": 9.567962122706648,
+      "grad_norm": 0.008032439276576042,
+      "learning_rate": 1.1318155524998597e-07,
+      "loss": 0.0002,
+      "step": 97000
+    },
+    {
+      "epoch": 9.617281515091735,
+      "grad_norm": 0.00466285552829504,
+      "learning_rate": 7.889703876469978e-08,
+      "loss": 0.0002,
+      "step": 97500
+    },
+    {
+      "epoch": 9.617281515091735,
+      "eval_all_ligands_equal": 0.5960516997167139,
+      "eval_e3_equal": 0.8272839943342776,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9888456090651558,
+      "eval_e3_heavy_atoms_difference": 0.1958215297450425,
+      "eval_e3_heavy_atoms_difference_norm": 0.0009669152686349697,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9888456090651558,
+      "eval_has_all_attachment_points": 0.9904390934844193,
+      "eval_has_three_substructures": 0.9994688385269122,
+      "eval_heavy_atoms_difference": 4.558870396600566,
+      "eval_heavy_atoms_difference_norm": 0.06187204058293781,
+      "eval_linker_equal": 0.8502124645892352,
+      "eval_linker_graph_edit_distance": 3.0984419263456094e+61,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9969015580736544,
+      "eval_linker_heavy_atoms_difference": 0.19821175637393768,
+      "eval_linker_heavy_atoms_difference_norm": 0.0008445360181919918,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9969015580736544,
+      "eval_loss": 0.39866751432418823,
+      "eval_num_fragments": 3.0003541076487252,
+      "eval_poi_equal": 0.790899433427762,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9602514164305949,
+      "eval_poi_heavy_atoms_difference": 1.1934313031161472,
+      "eval_poi_heavy_atoms_difference_norm": 0.03607653809133688,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9602514164305949,
+      "eval_reassembly": 0.6026026912181303,
+      "eval_reassembly_nostereo": 0.6339412181303116,
+      "eval_runtime": 2314.9163,
+      "eval_samples_per_second": 4.88,
+      "eval_steps_per_second": 0.076,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9475920679886686,
+      "step": 97500
+    },
+    {
+      "epoch": 9.66660090747682,
+      "grad_norm": 0.0011036242358386517,
+      "learning_rate": 5.077374756873865e-08,
+      "loss": 0.0002,
+      "step": 98000
+    },
+    {
+      "epoch": 9.715920299861907,
+      "grad_norm": 0.0038821862544864416,
+      "learning_rate": 2.8818634561947334e-08,
+      "loss": 0.0002,
+      "step": 98500
+    },
+    {
+      "epoch": 9.765239692246992,
+      "grad_norm": 0.008210109546780586,
+      "learning_rate": 1.3037127690179729e-08,
+      "loss": 0.0002,
+      "step": 99000
+    },
+    {
+      "epoch": 9.814559084632076,
+      "grad_norm": 0.004091315437108278,
+      "learning_rate": 3.44616961621691e-09,
+      "loss": 0.0002,
+      "step": 99500
+    },
+    {
+      "epoch": 9.863878477017163,
+      "grad_norm": 0.0069528305903077126,
+      "learning_rate": 9.691585167725947e-12,
+      "loss": 0.0002,
+      "step": 100000
+    },
+    {
+      "epoch": 9.863878477017163,
+      "eval_all_ligands_equal": 0.5960516997167139,
+      "eval_e3_equal": 0.8272839943342776,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9886685552407932,
+      "eval_e3_heavy_atoms_difference": 0.19573300283286119,
+      "eval_e3_heavy_atoms_difference_norm": 0.001114461012300652,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9886685552407932,
+      "eval_has_all_attachment_points": 0.9904390934844193,
+      "eval_has_three_substructures": 0.9994688385269122,
+      "eval_heavy_atoms_difference": 4.570024787535411,
+      "eval_heavy_atoms_difference_norm": 0.0620230810464109,
+      "eval_linker_equal": 0.8502124645892352,
+      "eval_linker_graph_edit_distance": 3.009915014164306e+61,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9969900849858357,
+      "eval_linker_heavy_atoms_difference": 0.19785764872521247,
+      "eval_linker_heavy_atoms_difference_norm": 0.0009133688030441173,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9969900849858357,
+      "eval_loss": 0.3986365497112274,
+      "eval_num_fragments": 3.0003541076487252,
+      "eval_poi_equal": 0.7909879603399433,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9601628895184136,
+      "eval_poi_heavy_atoms_difference": 1.1950247875354107,
+      "eval_poi_heavy_atoms_difference_norm": 0.03611503910692388,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9601628895184136,
+      "eval_reassembly": 0.6025141643059491,
+      "eval_reassembly_nostereo": 0.6337641643059491,
+      "eval_runtime": 2312.0798,
+      "eval_samples_per_second": 4.886,
+      "eval_steps_per_second": 0.077,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.947415014164306,
+      "step": 100000
     }
   ],
   "logging_steps": 500,
@@ -1939,12 +2093,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.292450664628905e+17,
+  "total_flos": 8.728946664336123e+17,
   "train_batch_size": 128,
   "trial_name": null,
   "trial_params": null