Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:776e810dd01ff44cd0ae5258784cc7a4fb43912af9e20ebd4fb14a16e6edf73a
+oid sha256:09bed90a49c49b3fad611ce85fb8a21b0919f4eb6e422fb98d39d7690d832b5a
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52449a96f9d7515622a053360c3cb75fff069929f11d88163ad9b3f90c1e0a21
+oid sha256:1ca2e207cc6e66faad7944c59b7e64942db2ef7028b0eecb3caacea355d4c7a2
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43f004ece9bfea3a082d65f9d4e84c653ebe3edebf4e07157b3b5a118e85e9b9
+oid sha256:713ae26eaf141bc0e0de8690938cd3c2675579894b6356633014c19e22d4be24
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77441d8a72ed30d9525730698a7169f1229749dee2b43dbae89633553d79ba30
+oid sha256:4da18a6f8a6db8955c278a8d0efe2d6193cd4355e0f3807dd771856fd055aa46
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.003977021652673442,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=24-learning_rate=1.2e-05-warmup_ratio=0.020-num_cycles=8.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.16977463543968183,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=24-learning_rate=1.2e-05-warmup_ratio=0.020-num_cycles=8.500/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8003535130357932,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.47750329971313477,
+      "learning_rate": 1.4564019408024557e-06,
+      "loss": 0.1274,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.5653688311576843,
+      "learning_rate": 9.485138751690928e-06,
+      "loss": 0.1155,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.409018874168396,
+      "learning_rate": 4.151505405056519e-06,
+      "loss": 0.1033,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.3600127398967743,
+      "learning_rate": 6.401994063154483e-06,
+      "loss": 0.0914,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.35262298583984375,
+      "learning_rate": 7.3983813284505514e-06,
+      "loss": 0.0845,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.06000883782589483,
+      "eval_e3_equal": 0.5713654441007512,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9953159522757402,
+      "eval_e3_heavy_atoms_difference": -0.2897039328325232,
+      "eval_e3_heavy_atoms_difference_norm": -0.025315724834858295,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9953159522757402,
+      "eval_has_all_attachment_points": 0.9748121961997349,
+      "eval_has_three_substructures": 0.9978789217852408,
+      "eval_heavy_atoms_difference": 10.714714980114891,
+      "eval_heavy_atoms_difference_norm": 0.14045413939927937,
+      "eval_linker_equal": 0.11833848873177198,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9950508174988952,
+      "eval_linker_heavy_atoms_difference": 1.941140079540433,
+      "eval_linker_heavy_atoms_difference_norm": 0.062388143869008136,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9950508174988952,
+      "eval_loss": 0.31332647800445557,
+      "eval_num_fragments": 2.999116217410517,
+      "eval_poi_equal": 0.5427308882015024,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.8867874502872294,
+      "eval_poi_heavy_atoms_difference": 3.6543526292532036,
+      "eval_poi_heavy_atoms_difference_norm": 0.11226265463872084,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.8867874502872294,
+      "eval_reassembly": 0.0619531595227574,
+      "eval_reassembly_nostereo": 0.07423773751657092,
+      "eval_runtime": 1499.315,
+      "eval_samples_per_second": 7.547,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8828104286345559,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.32262352108955383,
+      "learning_rate": 3.2276089371507582e-06,
+      "loss": 0.0731,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.38012757897377014,
+      "learning_rate": 1.0269327593973854e-05,
+      "loss": 0.0697,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.343169629573822,
+      "learning_rate": 8.68973314497152e-07,
+      "loss": 0.0593,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.3316091299057007,
+      "learning_rate": 1.1944053253900434e-05,
+      "loss": 0.0569,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.3776050806045532,
+      "learning_rate": 0.0,
+      "loss": 0.0496,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.16977463543968183,
+      "eval_e3_equal": 0.6699955810870526,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9946973044631021,
+      "eval_e3_heavy_atoms_difference": -0.26548828988068934,
+      "eval_e3_heavy_atoms_difference_norm": -0.023844943994185776,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9946973044631021,
+      "eval_has_all_attachment_points": 0.979584622182943,
+      "eval_has_three_substructures": 0.9983208130799823,
+      "eval_heavy_atoms_difference": 7.734158197083517,
+      "eval_heavy_atoms_difference_norm": 0.10348961558522504,
+      "eval_linker_equal": 0.273972602739726,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9957578435704817,
+      "eval_linker_heavy_atoms_difference": 1.027220503756076,
+      "eval_linker_heavy_atoms_difference_norm": 0.020864102938558286,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9957578435704817,
+      "eval_loss": 0.30179664492607117,
+      "eval_num_fragments": 2.9995581087052585,
+      "eval_poi_equal": 0.6329650905877154,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9117101193106496,
+      "eval_poi_heavy_atoms_difference": 2.6090145824127267,
+      "eval_poi_heavy_atoms_difference_norm": 0.08316842183865562,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9117101193106496,
+      "eval_reassembly": 0.172779496243924,
+      "eval_reassembly_nostereo": 0.19513919575784358,
+      "eval_runtime": 1497.157,
+      "eval_samples_per_second": 7.558,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9069376933274415,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=24-learning_rate=1.2e-05-warmup_ratio=0.020-num_cycles=8.500",
   "trial_params": {