Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78075f2103ed4a42dedc5a1d820d6521381eafd4553cdaa9f67f5ba464aa7ba7
+oid sha256:7bf9c6ea4115da7ecb26efa2880f84d13e203883e1744e63861c53658f9277f2
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:806e1398395c6c137c0587f13a844c7769ca3e41f3583278b75453cb7c3c3776
+oid sha256:2b6e2d89d31809aafc6d9f1a3d4b804a258907c5f3f1057e64ab7c0b2b18e13b
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:972803055246006a6d1cdf92a57afeed4f447ae6fbc63a34061227ad14fecd46
+oid sha256:e864df1cdce41b5f6008f2f3056157fe1a59741e45538c35750dad335d89adcf
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38a7bd14f8b8d9e4d201cd3bf5e6023863e154cc36dcada795ee0cf1077fdf97
+oid sha256:3b2e2f4a4d698b32d29e4f198023bd7f534f0e63616c12fe3bd5d6eef023d892
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.007247017233760495,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.16827220503756077,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8760936809544851,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.5318701863288879,
+      "learning_rate": 1.1237446278275383e-05,
+      "loss": 0.1316,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.3978229761123657,
+      "learning_rate": 5.625686628642194e-06,
+      "loss": 0.1163,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.3678536117076874,
+      "learning_rate": 3.7307809795190627e-06,
+      "loss": 0.1024,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.4965149462223053,
+      "learning_rate": 1.2452489342403467e-05,
+      "loss": 0.0972,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.4630180895328522,
+      "learning_rate": 3.319116134821198e-08,
+      "loss": 0.0827,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.061511268228015906,
+      "eval_e3_equal": 0.5425541316836058,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9982324348210341,
+      "eval_e3_heavy_atoms_difference": -0.41617322138753865,
+      "eval_e3_heavy_atoms_difference_norm": -0.030251585246387123,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9982324348210341,
+      "eval_has_all_attachment_points": 0.9847105612019443,
+      "eval_has_three_substructures": 0.9987627043747238,
+      "eval_heavy_atoms_difference": 6.157048166151127,
+      "eval_heavy_atoms_difference_norm": 0.0799003116743603,
+      "eval_linker_equal": 0.12072470172337604,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9971718957136544,
+      "eval_linker_heavy_atoms_difference": 1.4752982766239504,
+      "eval_linker_heavy_atoms_difference_norm": 0.026323008455953905,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9971718957136544,
+      "eval_loss": 0.31108492612838745,
+      "eval_num_fragments": 2.999646486964207,
+      "eval_poi_equal": 0.5619973486522315,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9346884666372073,
+      "eval_poi_heavy_atoms_difference": 1.8780380026513477,
+      "eval_poi_heavy_atoms_difference_norm": 0.053725194951032486,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9346884666372073,
+      "eval_reassembly": 0.0625718073353955,
+      "eval_reassembly_nostereo": 0.07618205921343349,
+      "eval_runtime": 1600.5588,
+      "eval_samples_per_second": 7.069,
+      "eval_steps_per_second": 0.111,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9327441449403446,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.36397504806518555,
+      "learning_rate": 1.1694228723322586e-05,
+      "loss": 0.0784,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.3220004737377167,
+      "learning_rate": 4.975250136973397e-06,
+      "loss": 0.0712,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.33015140891075134,
+      "learning_rate": 4.341505051882401e-06,
+      "loss": 0.0627,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.3355898857116699,
+      "learning_rate": 1.2100595958768545e-05,
+      "loss": 0.0608,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.2892158329486847,
+      "learning_rate": 0.0,
+      "loss": 0.0528,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.16827220503756077,
+      "eval_e3_equal": 0.639063190455148,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9983208130799823,
+      "eval_e3_heavy_atoms_difference": -0.2734423331860362,
+      "eval_e3_heavy_atoms_difference_norm": -0.024892349019047463,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9983208130799823,
+      "eval_has_all_attachment_points": 0.9885992045956694,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 4.554838709677419,
+      "eval_heavy_atoms_difference_norm": 0.058807906961281145,
+      "eval_linker_equal": 0.28501988510826337,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9977905435262925,
+      "eval_linker_heavy_atoms_difference": 0.91064958020327,
+      "eval_linker_heavy_atoms_difference_norm": 0.007958336622586505,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9977905435262925,
+      "eval_loss": 0.30263492465019226,
+      "eval_num_fragments": 2.999734865223155,
+      "eval_poi_equal": 0.632523199292974,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9508616880247459,
+      "eval_poi_heavy_atoms_difference": 1.4015908086610693,
+      "eval_poi_heavy_atoms_difference_norm": 0.0383393486603852,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9508616880247459,
+      "eval_reassembly": 0.1715422006186478,
+      "eval_reassembly_nostereo": 0.1992045956694653,
+      "eval_runtime": 1518.0837,
+      "eval_samples_per_second": 7.453,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9489173663278834,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.364959997072179e+16,
+  "total_flos": 8.729673191023411e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500",
   "trial_params": {