Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a45b371b49d393e7b2151d53d2b27c76511a92e0de3872c21a019811eadd052
+oid sha256:7c8f98e67975d4b1c7acc39b1d8bc450939844e83f3c48d320ce935c9b989066
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2360628fad224cd416ebca5be014502c46937c0891297d64ecd94129741bf9ae
+oid sha256:41c593d0ea8d92085799fad0ae6affc3c045e91d6835819162a8127478c2ca17
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a773ec5dc9d8e01fdd06c36cb7c97677e33c03e44ee1d619e1b6ea8cf7aaaa6
+oid sha256:e366becca1198279a262244c18efaa2b05b928046d04023485d26d11f781ec0a
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c20dabbe2ff644f70a12804f919fe238c74ed2e7c6d2f4380fe1fe1953807210
+oid sha256:7b28b4601a5497ced593015fbd7f37e3a7b9cde43ba0f7bd18d5967f1c63ea91
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.5283252319929297,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=14-learning_rate=9.3e-05-warmup_ratio=0.020-min_lr=0.000-factor=0.820/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.55324790101635,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=14-learning_rate=9.3e-05-warmup_ratio=0.020-min_lr=0.000-factor=0.820/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9269995581087053,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.09683173894882202,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0044,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.07747574895620346,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0043,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.07926973700523376,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0052,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.0817081481218338,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0036,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.10049362480640411,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0036,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.5418471056120194,
+      "eval_e3_equal": 0.8101634997790543,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9891294741493593,
+      "eval_e3_heavy_atoms_difference": 0.5689792311091472,
+      "eval_e3_heavy_atoms_difference_norm": 0.012787717815485097,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9891294741493593,
+      "eval_has_all_attachment_points": 0.9857711003093239,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 7.5807335395492705,
+      "eval_heavy_atoms_difference_norm": 0.09946871788139681,
+      "eval_linker_equal": 0.7713654441007513,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9926646045072912,
+      "eval_linker_heavy_atoms_difference": 0.691117984975696,
+      "eval_linker_heavy_atoms_difference_norm": 0.025806209525107107,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9926646045072912,
+      "eval_loss": 0.3165743052959442,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.7655324790101635,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.930181175430844,
+      "eval_poi_heavy_atoms_difference": 2.1524524966858154,
+      "eval_poi_heavy_atoms_difference_norm": 0.06685042263605857,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.930181175430844,
+      "eval_reassembly": 0.5498895271763147,
+      "eval_reassembly_nostereo": 0.5723376049491825,
+      "eval_runtime": 1500.4697,
+      "eval_samples_per_second": 7.541,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9143614670790985,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.07925644516944885,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.004,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.0687689334154129,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.003,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.0820043683052063,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0029,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.12879632413387299,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0031,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.09962965548038483,
+      "learning_rate": 9.310557245969489e-05,
+      "loss": 0.0039,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.55324790101635,
+      "eval_e3_equal": 0.8146707909854176,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9780821917808219,
+      "eval_e3_heavy_atoms_difference": 0.6205921343349536,
+      "eval_e3_heavy_atoms_difference_norm": 0.01700516523969378,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9780821917808219,
+      "eval_has_all_attachment_points": 0.9807335395492709,
+      "eval_has_three_substructures": 0.9988510826336722,
+      "eval_heavy_atoms_difference": 8.770835174547061,
+      "eval_heavy_atoms_difference_norm": 0.11466190179254125,
+      "eval_linker_equal": 0.7853292090145824,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9953159522757402,
+      "eval_linker_heavy_atoms_difference": 0.4824569155987627,
+      "eval_linker_heavy_atoms_difference_norm": 0.00924100224677428,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9953159522757402,
+      "eval_loss": 0.3139532804489136,
+      "eval_num_fragments": 2.9992045956694655,
+      "eval_poi_equal": 0.7662395050817499,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9167476800707026,
+      "eval_poi_heavy_atoms_difference": 2.677507733097658,
+      "eval_poi_heavy_atoms_difference_norm": 0.08437848660906058,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9167476800707026,
+      "eval_reassembly": 0.5614670790985418,
+      "eval_reassembly_nostereo": 0.5764030048608042,
+      "eval_runtime": 1544.4322,
+      "eval_samples_per_second": 7.326,
+      "eval_steps_per_second": 0.115,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8949182501104728,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=14-learning_rate=9.3e-05-warmup_ratio=0.020-min_lr=0.000-factor=0.820",
   "trial_params": {