Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a261f21c4798690ab57bcf5ffc75cb635a31db58c4ea1604e9353a1c108bc5d7
+oid sha256:95a1e4676557631a08fbea97e286406398a14c1af940a0fd48c328d0cce155f6
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78a7433f411b7cc9960ad60d7ebc407c5ec0e8982ff1dd8b9b86468cc9e119f1
+oid sha256:4b77c467ed9e190a97774d0e80cea3bc2ceb7238651e85ac65934510d4b88bbc
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6f438cb0fdeca1df93fcdce1edc2b32e96f7f0d6712a03bb6357822cddd5336
+oid sha256:b85cd0360d9408a97f595f91b2c2654542a0ab2fe6f214339ae57f297c8e0f4e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea5c1aec2970c9bb32855321c898130406a98d15b07950cfe8fd52372466f192
+oid sha256:89b7730f1bd274eb55646d3cd6eb24e3e89d0753098d21c1629934f850c0c956
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.5236411842686699,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=3-learning_rate=1.1e-04-warmup_ratio=0.050-min_lr=0.000-factor=0.850/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5623508616880247,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=3-learning_rate=1.1e-04-warmup_ratio=0.050-min_lr=0.000-factor=0.850/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.921608484312859,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.11470098048448563,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0042,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.11638347059488297,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0042,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.07129918038845062,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0059,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.044687844812870026,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0034,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.0823349729180336,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0031,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.5498895271763147,
+      "eval_e3_equal": 0.8119310649580204,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9771100309323907,
+      "eval_e3_heavy_atoms_difference": 0.8516129032258064,
+      "eval_e3_heavy_atoms_difference_norm": 0.02148143499541932,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9771100309323907,
+      "eval_has_all_attachment_points": 0.9779938135218736,
+      "eval_has_three_substructures": 0.9984091913389306,
+      "eval_heavy_atoms_difference": 7.066725585505965,
+      "eval_heavy_atoms_difference_norm": 0.09269971150883742,
+      "eval_linker_equal": 0.7787892178524083,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9923994697304463,
+      "eval_linker_heavy_atoms_difference": 0.4481661511268228,
+      "eval_linker_heavy_atoms_difference_norm": 0.006821181166453098,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9923994697304463,
+      "eval_loss": 0.31453314423561096,
+      "eval_num_fragments": 2.9998232434821035,
+      "eval_poi_equal": 0.7662395050817499,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9374281926646045,
+      "eval_poi_heavy_atoms_difference": 1.763057887759611,
+      "eval_poi_heavy_atoms_difference_norm": 0.05254183317720355,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9374281926646045,
+      "eval_reassembly": 0.5576668139637649,
+      "eval_reassembly_nostereo": 0.5741051701281484,
+      "eval_runtime": 1494.0608,
+      "eval_samples_per_second": 7.573,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9133009279717189,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.0743485540151596,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0034,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.08227942138910294,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0035,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.07675650715827942,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.004,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.060432229191064835,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.003,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.06841175258159637,
+      "learning_rate": 0.00010780369551982127,
+      "loss": 0.0024,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5623508616880247,
+      "eval_e3_equal": 0.8172337604949182,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9732213875386655,
+      "eval_e3_heavy_atoms_difference": 0.8075121520106054,
+      "eval_e3_heavy_atoms_difference_norm": 0.02178934615008687,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9732213875386655,
+      "eval_has_all_attachment_points": 0.9811754308440124,
+      "eval_has_three_substructures": 0.9995581087052585,
+      "eval_heavy_atoms_difference": 7.874414494034467,
+      "eval_heavy_atoms_difference_norm": 0.10424160867591652,
+      "eval_linker_equal": 0.8018559434379142,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9961997348652232,
+      "eval_linker_heavy_atoms_difference": 0.41741051701281484,
+      "eval_linker_heavy_atoms_difference_norm": 0.008801670643119743,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9961997348652232,
+      "eval_loss": 0.3179270923137665,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.771895713654441,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9324790101634998,
+      "eval_poi_heavy_atoms_difference": 1.9437030490499336,
+      "eval_poi_heavy_atoms_difference_norm": 0.059822219414631145,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9324790101634998,
+      "eval_reassembly": 0.5704816615112682,
+      "eval_reassembly_nostereo": 0.5938135218736191,
+      "eval_runtime": 1488.581,
+      "eval_samples_per_second": 7.601,
+      "eval_steps_per_second": 0.119,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9044631020768891,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=3-learning_rate=1.1e-04-warmup_ratio=0.050-min_lr=0.000-factor=0.850",
   "trial_params": {