Training in progress, step 100000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2990bb2d35a49909a89bff442f13d61a7417a5e079a6cf8e28923274cb2002e
+oid sha256:e31236a2e8eaca603a628b01a63ff114c74ee17c0e0f1af6b8f0e39796df19b2
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c680bd4586644b3bdf71bbeb0044558e338627a32760a0d0d947284db698fd1
+oid sha256:d7ce76b1960b19d103c48bee5e71d5ca001a4f7681c94f1226c0ad62885dc50c
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e040256c3112adc3eb4fea1f73182fd34c1abe5a766c4391a9a33b9c56f48dd
+oid sha256:4e8ee84d127f5376e5feb3d3923bdaaf9813bd029048a032bf9a0241a1855c5f
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e88ab7566e2f176bfbc5e980b089e6c7a791d55062e217ce8bc66f628d299897
+oid sha256:82fa0ea10636b8e5586129e098a1478a1e39b1db76b450cb5d6f92f0aab0da80
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.5946352691218131,
   "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-rand-smiles/checkpoint-80000",
-  "epoch": 9.370684553166305,
+  "epoch": 9.863878477017163,
   "eval_steps": 2500,
-  "global_step": 95000,
+  "global_step": 100000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1925,6 +1925,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9407754957507082,
       "step": 95000
+    },
+    {
+      "epoch": 9.420003945551391,
+      "grad_norm": 0.024864112958312035,
+      "learning_rate": 1.5046603559991678e-05,
+      "loss": 0.0005,
+      "step": 95500
+    },
+    {
+      "epoch": 9.469323337936476,
+      "grad_norm": 0.006345201283693314,
+      "learning_rate": 5.659679305879944e-06,
+      "loss": 0.0004,
+      "step": 96000
+    },
+    {
+      "epoch": 9.518642730321563,
+      "grad_norm": 0.02008950337767601,
+      "learning_rate": 5.556772733849275e-07,
+      "loss": 0.0004,
+      "step": 96500
+    },
+    {
+      "epoch": 9.567962122706648,
+      "grad_norm": 0.026644747704267502,
+      "learning_rate": 4.9135119122348355e-05,
+      "loss": 0.0009,
+      "step": 97000
+    },
+    {
+      "epoch": 9.617281515091735,
+      "grad_norm": 0.026153231039643288,
+      "learning_rate": 4.348118315064496e-05,
+      "loss": 0.0009,
+      "step": 97500
+    },
+    {
+      "epoch": 9.617281515091735,
+      "eval_all_ligands_equal": 0.5872875354107648,
+      "eval_e3_equal": 0.8247167138810199,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9954851274787535,
+      "eval_e3_heavy_atoms_difference": 0.2013987252124646,
+      "eval_e3_heavy_atoms_difference_norm": 0.0014598404087872825,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9954851274787535,
+      "eval_has_all_attachment_points": 0.9920325779036827,
+      "eval_has_three_substructures": 0.9999114730878187,
+      "eval_heavy_atoms_difference": 4.997875354107649,
+      "eval_heavy_atoms_difference_norm": 0.06514101315267158,
+      "eval_linker_equal": 0.8380842776203966,
+      "eval_linker_graph_edit_distance": 3.009915014164306e+61,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9969900849858357,
+      "eval_linker_heavy_atoms_difference": 0.1986543909348442,
+      "eval_linker_heavy_atoms_difference_norm": -0.0003615286386425098,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9969900849858357,
+      "eval_loss": 0.34618067741394043,
+      "eval_num_fragments": 2.9999114730878187,
+      "eval_poi_equal": 0.7862960339943342,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9459100566572238,
+      "eval_poi_heavy_atoms_difference": 1.5948123229461757,
+      "eval_poi_heavy_atoms_difference_norm": 0.04803252024322969,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9459100566572238,
+      "eval_reassembly": 0.5945467422096318,
+      "eval_reassembly_nostereo": 0.6288951841359773,
+      "eval_runtime": 2334.8639,
+      "eval_samples_per_second": 4.838,
+      "eval_steps_per_second": 0.076,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9390049575070821,
+      "step": 97500
+    },
+    {
+      "epoch": 9.66660090747682,
+      "grad_norm": 0.025788171216845512,
+      "learning_rate": 3.373458133076218e-05,
+      "loss": 0.0006,
+      "step": 98000
+    },
+    {
+      "epoch": 9.715920299861907,
+      "grad_norm": 0.021747371181845665,
+      "learning_rate": 2.205370277008977e-05,
+      "loss": 0.0005,
+      "step": 98500
+    },
+    {
+      "epoch": 9.765239692246992,
+      "grad_norm": 0.011911082081496716,
+      "learning_rate": 1.1025282980761118e-05,
+      "loss": 0.0005,
+      "step": 99000
+    },
+    {
+      "epoch": 9.814559084632076,
+      "grad_norm": 0.011921499855816364,
+      "learning_rate": 3.0915702032295766e-06,
+      "loss": 0.0004,
+      "step": 99500
+    },
+    {
+      "epoch": 9.863878477017163,
+      "grad_norm": 0.010130259208381176,
+      "learning_rate": 9.488497454016032e-09,
+      "loss": 0.0004,
+      "step": 100000
+    },
+    {
+      "epoch": 9.863878477017163,
+      "eval_all_ligands_equal": 0.5946352691218131,
+      "eval_e3_equal": 0.8281692634560907,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9934490084985835,
+      "eval_e3_heavy_atoms_difference": 0.18829674220963172,
+      "eval_e3_heavy_atoms_difference_norm": 0.002624594050018952,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9934490084985835,
+      "eval_has_all_attachment_points": 0.9938916430594901,
+      "eval_has_three_substructures": 0.9997344192634561,
+      "eval_heavy_atoms_difference": 4.238225920679887,
+      "eval_heavy_atoms_difference_norm": 0.05702418286890371,
+      "eval_linker_equal": 0.8511862606232294,
+      "eval_linker_graph_edit_distance": Infinity,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9983179886685553,
+      "eval_linker_heavy_atoms_difference": 0.20122167138810199,
+      "eval_linker_heavy_atoms_difference_norm": 0.0036037839164644943,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9983179886685553,
+      "eval_loss": 0.3583788275718689,
+      "eval_num_fragments": 2.999734419263456,
+      "eval_poi_equal": 0.7926699716713881,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9567103399433428,
+      "eval_poi_heavy_atoms_difference": 1.3881905099150142,
+      "eval_poi_heavy_atoms_difference_norm": 0.044135254224856695,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9567103399433428,
+      "eval_reassembly": 0.6014518413597734,
+      "eval_reassembly_nostereo": 0.6357117563739377,
+      "eval_runtime": 2331.2375,
+      "eval_samples_per_second": 4.845,
+      "eval_steps_per_second": 0.076,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9492740793201133,
+      "step": 100000
     }
   ],
   "logging_steps": 500,
@@ -1939,12 +2093,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.292450664628905e+17,
+  "total_flos": 8.728946664336123e+17,
   "train_batch_size": 128,
   "trial_name": null,
   "trial_params": null