Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12062b000e1cb8b44553fac4d3e1162aea140c509f7c2dbcfea2aca7cce00046
+oid sha256:a608ff64b48e48b57a79452b34567acbe67a23ada467648f1a5f0c1d6bdcd248
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a20d79c3209721c19c5ab15349c56358582b4c970a52de9820dfcd02eb0bf0e
+oid sha256:4e6cee46d5bf233e52f8dbeb5bd38c5f43127506bc7454b01ea48c9950451f9e
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bbbef9c5a6186d36adcd5731e7b55b4690c7b3d086f02ba830c3a77bd5cd486
+oid sha256:8c6301d04d7e302f9a20d362ab718c250a725ee45dbb7a65cb9261444491d2da
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:655073625197a7e4274d7f9548f06c1fcefb405e98bb15971106f0841041af52
+oid sha256:e48fb3aab7d48713e9e77878b87e09382b75a08326ff8fa709483f3a43dd943f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.11842686699072029,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4.000/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.4196199734865223,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4.000/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8695536897923111,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.27302974462509155,
+      "learning_rate": 2.554475398889662e-05,
+      "loss": 0.0444,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.23224854469299316,
+      "learning_rate": 1.0840465507701789e-05,
+      "loss": 0.0319,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.23858347535133362,
+      "learning_rate": 7.146086424448981e-09,
+      "loss": 0.0252,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.36267736554145813,
+      "learning_rate": 9.989074812002888e-06,
+      "loss": 0.0243,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.31655964255332947,
+      "learning_rate": 2.517362368235566e-05,
+      "loss": 0.024,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.3515687140963323,
+      "eval_e3_equal": 0.7513035793194874,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.991427308882015,
+      "eval_e3_heavy_atoms_difference": 0.1234644277507733,
+      "eval_e3_heavy_atoms_difference_norm": -0.003241441692798868,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.991427308882015,
+      "eval_has_all_attachment_points": 0.9791427308882015,
+      "eval_has_three_substructures": 0.9984091913389306,
+      "eval_heavy_atoms_difference": 8.571277065841803,
+      "eval_heavy_atoms_difference_norm": 0.11408275021017346,
+      "eval_linker_equal": 0.526646045072912,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9937251436146708,
+      "eval_linker_heavy_atoms_difference": 0.15510384445426426,
+      "eval_linker_heavy_atoms_difference_norm": -0.03091358477610182,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9937251436146708,
+      "eval_loss": 0.29450246691703796,
+      "eval_num_fragments": 3.0001767565178965,
+      "eval_poi_equal": 0.6740609809986743,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.8986301369863013,
+      "eval_poi_heavy_atoms_difference": 2.8806893504197966,
+      "eval_poi_heavy_atoms_difference_norm": 0.09157872702647427,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.8986301369863013,
+      "eval_reassembly": 0.35536897923110916,
+      "eval_reassembly_nostereo": 0.3800265134776845,
+      "eval_runtime": 1620.5079,
+      "eval_samples_per_second": 6.982,
+      "eval_steps_per_second": 0.109,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8885550154661953,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.2239069789648056,
+      "learning_rate": 2.181080214858652e-05,
+      "loss": 0.0214,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.19752831757068634,
+      "learning_rate": 5.160363419470711e-06,
+      "loss": 0.0154,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.192670539021492,
+      "learning_rate": 1.265080700050313e-06,
+      "loss": 0.0129,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.19295284152030945,
+      "learning_rate": 1.621752404699275e-05,
+      "loss": 0.0131,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.23245178163051605,
+      "learning_rate": 2.663073667910919e-05,
+      "loss": 0.0142,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.4196199734865223,
+      "eval_e3_equal": 0.7765797613787009,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9922227132125497,
+      "eval_e3_heavy_atoms_difference": 0.3922227132125497,
+      "eval_e3_heavy_atoms_difference_norm": 0.006326123786719304,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9922227132125497,
+      "eval_has_all_attachment_points": 0.9733097657976137,
+      "eval_has_three_substructures": 0.9983208130799823,
+      "eval_heavy_atoms_difference": 9.077684489615555,
+      "eval_heavy_atoms_difference_norm": 0.12011828880330795,
+      "eval_linker_equal": 0.6334953601414052,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9942554131683606,
+      "eval_linker_heavy_atoms_difference": 0.6975695978789218,
+      "eval_linker_heavy_atoms_difference_norm": 0.015551211535156739,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9942554131683606,
+      "eval_loss": 0.30925416946411133,
+      "eval_num_fragments": 3.000618647812638,
+      "eval_poi_equal": 0.6826336721166593,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9060539107379585,
+      "eval_poi_heavy_atoms_difference": 3.0226248342907645,
+      "eval_poi_heavy_atoms_difference_norm": 0.09735135794581425,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9060539107379585,
+      "eval_reassembly": 0.4265134776844896,
+      "eval_reassembly_nostereo": 0.4486964206805126,
+      "eval_runtime": 1487.364,
+      "eval_samples_per_second": 7.607,
+      "eval_steps_per_second": 0.119,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8972160848431285,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4.000",
   "trial_params": {