Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63a116401022f996157f07351ab506fbaf81247489fef998567d934ba30460a9
+oid sha256:1e012f5a2962b1055e7e91314208508fb0e2b5839688d9c61c30ead3cb378e53
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edc1fa1aff1223cca44f2c4d928e15efab0ed8f3681d62b18a000ceaf93f8e89
+oid sha256:b28a919b0dae87b17bfb875069130ea193e1bbe0b825ed2e91ca448339d232fd
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66e18f2faebccbc562f45f56c88b60e92b37c83eedbc58d95807945752a79d43
+oid sha256:df90c14ac5ab218269cb46bc8c052aca82e45e962bacfffe6b5d990927a38a8d
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02ed5449d5500edddbfb81c169a4337ca29e83c1a75cfa85d8c9e7460dc3ee9e
+oid sha256:8d92c5a2febec6a3debc54e07f631908af44ee341af8b138c2ac3072c6cfbde8
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.0,
   "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=2/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.19938135218736192,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 1.1991398334503174,
+      "learning_rate": 1.5828157639979357e-06,
+      "loss": 0.7666,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 1.2663832902908325,
+      "learning_rate": 1.473107625487659e-06,
+      "loss": 0.6824,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 1.2193111181259155,
+      "learning_rate": 1.296960161557218e-06,
+      "loss": 0.6184,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 1.3224248886108398,
+      "learning_rate": 1.071963064180241e-06,
+      "loss": 0.5711,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 1.1388825178146362,
+      "learning_rate": 8.205840405238564e-07,
+      "loss": 0.5359,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 8.837825894829872e-05,
+      "eval_e3_equal": 0.07414935925762263,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9784357048166151,
+      "eval_e3_heavy_atoms_difference": -0.01643835616438356,
+      "eval_e3_heavy_atoms_difference_norm": -0.041997529417292574,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9784357048166151,
+      "eval_has_all_attachment_points": 0.7959346000883782,
+      "eval_has_three_substructures": 0.9798497569597879,
+      "eval_heavy_atoms_difference": 59.51330092797172,
+      "eval_heavy_atoms_difference_norm": 0.7667716663424441,
+      "eval_linker_equal": 0.0017675651789659744,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9551922227132126,
+      "eval_linker_heavy_atoms_difference": 7.438002651347769,
+      "eval_linker_heavy_atoms_difference_norm": 0.3411107847976279,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9551922227132126,
+      "eval_loss": 0.6140255928039551,
+      "eval_num_fragments": 3.007423773751657,
+      "eval_poi_equal": 0.005921343349536014,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.3249668581528944,
+      "eval_poi_heavy_atoms_difference": 24.587361908970394,
+      "eval_poi_heavy_atoms_difference_norm": 0.7407628377914883,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.3249668581528944,
+      "eval_reassembly": 8.837825894829872e-05,
+      "eval_reassembly_nostereo": 8.837825894829872e-05,
+      "eval_runtime": 1405.5464,
+      "eval_samples_per_second": 8.05,
+      "eval_steps_per_second": 0.126,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.3149801148917366,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 1.2519776821136475,
+      "learning_rate": 5.679252379371353e-07,
+      "loss": 0.5131,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 1.2744039297103882,
+      "learning_rate": 3.392165996322104e-07,
+      "loss": 0.4951,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.9478299617767334,
+      "learning_rate": 1.5729645895619162e-07,
+      "loss": 0.4866,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 1.2296202182769775,
+      "learning_rate": 4.033095447780681e-08,
+      "loss": 0.4822,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 1.018215537071228,
+      "learning_rate": 0.0,
+      "loss": 0.4806,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.08387096774193549,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9821475916924437,
+      "eval_e3_heavy_atoms_difference": -0.6608926204153778,
+      "eval_e3_heavy_atoms_difference_norm": -0.06311356185857149,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9821475916924437,
+      "eval_has_all_attachment_points": 0.7999116217410517,
+      "eval_has_three_substructures": 0.9835616438356164,
+      "eval_heavy_atoms_difference": 55.602032699955814,
+      "eval_heavy_atoms_difference_norm": 0.7141083957383162,
+      "eval_linker_equal": 0.0016791869200176756,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9660627485638533,
+      "eval_linker_heavy_atoms_difference": 7.421387538665488,
+      "eval_linker_heavy_atoms_difference_norm": 0.34276215918626973,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9660627485638533,
+      "eval_loss": 0.5785325169563293,
+      "eval_num_fragments": 3.004065399911622,
+      "eval_poi_equal": 0.011135660627485638,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.3761378700839593,
+      "eval_poi_heavy_atoms_difference": 22.859920459566947,
+      "eval_poi_heavy_atoms_difference_norm": 0.6843588799912407,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.3761378700839593,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 1402.5909,
+      "eval_samples_per_second": 8.067,
+      "eval_steps_per_second": 0.126,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.36897923110914715,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=2",
   "trial_params": {