Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8fc98e3feff45adbfecc20c1e25564c6fab000d9f738e8e9feeeb22937177f8
+oid sha256:76153a28fb36deaf4840c2c04f9e1ed5199463d33a3e9bcab9472845b7663175
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dfa2f99fcfea5b880b3cdff501b6e446867748078e92f01efcf8350774632a4e
+oid sha256:c002d65a5534e904351be9f44157d8c02cc3ffe43d401f03775668ffb5b47558
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b011a31d42c1446fe5cc11f0c3078bae2b04dae6c0d5ab945e989a90dfcde070
+oid sha256:04eb79d8694bb3e55830f51c38772249e312badb231c8a7f663e515ae03b238d
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:418250b00e984c16e542a3795ab0a0ea5ca42ee5c239316091a0ed085900733e
+oid sha256:72c8681c447c584ebdea824a6f01669a89902ce302fb2e112482098313ece7b1
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.026513477684489615,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=20-learning_rate=8.1e-06-warmup_ratio=0.020-min_lr=0.000-factor=0.870/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.28493150684931506,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=20-learning_rate=8.1e-06-warmup_ratio=0.020-min_lr=0.000-factor=0.870/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8687582854617764,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.4395071268081665,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0916,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.4391575753688812,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0792,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.3817509412765503,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0689,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.39449697732925415,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0608,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.38708481192588806,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0535,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.15360141405214317,
+      "eval_e3_equal": 0.6699072028281042,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9961113566062748,
+      "eval_e3_heavy_atoms_difference": -0.004418912947414936,
+      "eval_e3_heavy_atoms_difference_norm": -0.013142892281090538,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9961113566062748,
+      "eval_has_all_attachment_points": 0.9812638091029606,
+      "eval_has_three_substructures": 0.9985859478568272,
+      "eval_heavy_atoms_difference": 9.262571807335396,
+      "eval_heavy_atoms_difference_norm": 0.12213830298561457,
+      "eval_linker_equal": 0.25992045956694654,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9955810870525851,
+      "eval_linker_heavy_atoms_difference": 1.6695536897923111,
+      "eval_linker_heavy_atoms_difference_norm": 0.05528848695760674,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9955810870525851,
+      "eval_loss": 0.3037106692790985,
+      "eval_num_fragments": 2.999646486964207,
+      "eval_poi_equal": 0.6048608042421564,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9078214759169244,
+      "eval_poi_heavy_atoms_difference": 3.1472381794078657,
+      "eval_poi_heavy_atoms_difference_norm": 0.1025784440640938,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9078214759169244,
+      "eval_reassembly": 0.15740167918692002,
+      "eval_reassembly_nostereo": 0.18099867432611577,
+      "eval_runtime": 1470.5736,
+      "eval_samples_per_second": 7.694,
+      "eval_steps_per_second": 0.12,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9031374281926646,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.29699942469596863,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0477,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.27733123302459717,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.042,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.382049560546875,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0374,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.2822589874267578,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.0338,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.26448339223861694,
+      "learning_rate": 8.081437055445882e-06,
+      "loss": 0.031,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.28493150684931506,
+      "eval_e3_equal": 0.7187803800265135,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9933716305788776,
+      "eval_e3_heavy_atoms_difference": 0.09703932832523199,
+      "eval_e3_heavy_atoms_difference_norm": -0.006468373051250263,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9933716305788776,
+      "eval_has_all_attachment_points": 0.9847989394608926,
+      "eval_has_three_substructures": 0.9981440565620857,
+      "eval_heavy_atoms_difference": 8.616526734423331,
+      "eval_heavy_atoms_difference_norm": 0.1142049484415829,
+      "eval_linker_equal": 0.43614670790985416,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9942554131683606,
+      "eval_linker_heavy_atoms_difference": 1.3867432611577553,
+      "eval_linker_heavy_atoms_difference_norm": 0.05168857272970736,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9942554131683606,
+      "eval_loss": 0.29920995235443115,
+      "eval_num_fragments": 2.9992045956694655,
+      "eval_poi_equal": 0.6578877596111357,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9113566062748564,
+      "eval_poi_heavy_atoms_difference": 2.860362350861688,
+      "eval_poi_heavy_atoms_difference_norm": 0.09209688691478542,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9113566062748564,
+      "eval_reassembly": 0.2906760936809545,
+      "eval_reassembly_nostereo": 0.32019443216968624,
+      "eval_runtime": 1456.936,
+      "eval_samples_per_second": 7.766,
+      "eval_steps_per_second": 0.121,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9041095890410958,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=20-learning_rate=8.1e-06-warmup_ratio=0.020-min_lr=0.000-factor=0.870",
   "trial_params": {