Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:639a5501bdc523e0441e31e1280002de7488fe1453462ae2e56eb0450c6992d4
+oid sha256:87bc001e53d8b757685c61a97c1467021cdf90723becdce2757020b3652b982a
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee9ea715118c6943aa86c3bb024cde64ac294ffea61c336a90468c037beb3f35
+oid sha256:90d1bf4eb6d1bb9271d5126b8f5a1c6dd17fbeae0f60a8ab31d61a5d4ebec11f
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:879472e287e065fa658f09ff323000650d929fe5965b3dc27832118b0efecf43
+oid sha256:f1d172b6355928a488e306b3c62c213f58a259204be7f1f4099838d768732abc
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ebcb3665db5fce40cf732f6c71e9b277aa58af5fb60f6220602d90c48ee3fc0
+oid sha256:e8949affef14be53a53951982b2c566ab947f5b43a210616fc35cd404aaa9e59
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4888201502430402,
   "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=5.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9680070702607159,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.11920630186796188,
+      "learning_rate": 0.00033406568194236334,
+      "loss": 0.0075,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.5241788625717163,
+      "learning_rate": 0.00036296734876936724,
+      "loss": 0.0502,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.3233102560043335,
+      "learning_rate": 2.0581448478610224e-05,
+      "loss": 0.0285,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.22846156358718872,
+      "learning_rate": 0.00018547973538572384,
+      "loss": 0.0155,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 1.3288661241531372,
+      "learning_rate": 0.0004345286134851596,
+      "loss": 0.0361,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.04896155545735749,
+      "eval_e3_equal": 0.40309323906319044,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9882456915598763,
+      "eval_e3_heavy_atoms_difference": 3.2123729562527616,
+      "eval_e3_heavy_atoms_difference_norm": 0.08806829276528134,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9882456915598763,
+      "eval_has_all_attachment_points": 0.9847105612019443,
+      "eval_has_three_substructures": 0.9967300044189129,
+      "eval_heavy_atoms_difference": 16.28095448519664,
+      "eval_heavy_atoms_difference_norm": 0.19772022011725113,
+      "eval_linker_equal": 0.19611135660627485,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9905435262925321,
+      "eval_linker_heavy_atoms_difference": 3.3720724701723377,
+      "eval_linker_heavy_atoms_difference_norm": 0.12169105760225543,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9905435262925321,
+      "eval_loss": 0.3659327030181885,
+      "eval_num_fragments": 2.997083517454706,
+      "eval_poi_equal": 0.37021652673442335,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.8987185152452497,
+      "eval_poi_heavy_atoms_difference": 5.3116217410517015,
+      "eval_poi_heavy_atoms_difference_norm": 0.13395514166933128,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.8987185152452497,
+      "eval_reassembly": 0.04993371630578878,
+      "eval_reassembly_nostereo": 0.058241272646928854,
+      "eval_runtime": 1444.1216,
+      "eval_samples_per_second": 7.835,
+      "eval_steps_per_second": 0.123,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8870525850640743,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.36206263303756714,
+      "learning_rate": 0.00012866172649195304,
+      "loss": 0.0394,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.40258172154426575,
+      "learning_rate": 5.274202368198822e-05,
+      "loss": 0.0172,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.4322950541973114,
+      "learning_rate": 0.00040158157654602796,
+      "loss": 0.0193,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.4600200355052948,
+      "learning_rate": 0.0002800483785552586,
+      "loss": 0.0508,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.38120532035827637,
+      "learning_rate": 0.0,
+      "loss": 0.0198,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.31842686699072026,
+      "eval_e3_equal": 0.6669023420238621,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9989394608926204,
+      "eval_e3_heavy_atoms_difference": 0.1460008837825895,
+      "eval_e3_heavy_atoms_difference_norm": -0.010953956022147847,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9989394608926204,
+      "eval_has_all_attachment_points": 0.9998232434821034,
+      "eval_has_three_substructures": 1.0,
+      "eval_heavy_atoms_difference": 0.7110914714980114,
+      "eval_heavy_atoms_difference_norm": 0.003622114487280091,
+      "eval_linker_equal": 0.5198409191338931,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9997348652231551,
+      "eval_linker_heavy_atoms_difference": 0.3863013698630137,
+      "eval_linker_heavy_atoms_difference_norm": -0.044767838129064215,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9997348652231551,
+      "eval_loss": 0.3589182198047638,
+      "eval_num_fragments": 3.0,
+      "eval_poi_equal": 0.6580645161290323,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.994078656650464,
+      "eval_poi_heavy_atoms_difference": -0.21882456915598764,
+      "eval_poi_heavy_atoms_difference_norm": -0.03121567410490773,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.994078656650464,
+      "eval_reassembly": 0.3226690234202386,
+      "eval_reassembly_nostereo": 0.34202386212991603,
+      "eval_runtime": 1587.8107,
+      "eval_samples_per_second": 7.126,
+      "eval_steps_per_second": 0.111,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9927529827662395,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=5.500",
   "trial_params": {