Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50011882f6e94485ecd911a464934f117f78064da0d44af2faa0a580d2dc69bc
+oid sha256:e3dec5292ae1c1392a3b2fd3086fca3e6132a40668d5f9e8722bb2702da71031
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d19696c9fc94f66c580b32203b546d48c2941b3566f35054fb4baaa703129588
+oid sha256:10d3ec786c2beb5fa0727629028ae6883e70683b9d54fdd5269035419b19d35f
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d22c10d7550e065c81ef680acc47320cb8967eab76c92c9bc164c3ac3c2f176
+oid sha256:a591a85ce4d04ee8922da4ecfae0211cffc7e3d570ec90d3e61a8896c0be80f4
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6828ba4e0a02ba8ef04fc15c3250ecdbde02ded1248c840d960be581184449ca
+oid sha256:3c1bedecd46112ca34d1e29a373aa4b494e52c2a218647a38e89a3652c149595
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.5443216968625718,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=6-learning_rate=1.6e-04-warmup_ratio=0.010-num_cycles=9.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5702165267344234,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=6-learning_rate=1.6e-04-warmup_ratio=0.010-num_cycles=9.500/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9205479452054794,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.12037888914346695,
+      "learning_rate": 0.00014497970292118344,
+      "loss": 0.004,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.0848931148648262,
+      "learning_rate": 1.1226339880940353e-05,
+      "loss": 0.0089,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.10188127309083939,
+      "learning_rate": 0.00015045283800984563,
+      "loss": 0.0036,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.05653466656804085,
+      "learning_rate": 6.381993872586385e-06,
+      "loss": 0.0091,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.10091005265712738,
+      "learning_rate": 0.00015464651640246716,
+      "loss": 0.0031,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.504463102076889,
+      "eval_e3_equal": 0.8027397260273973,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9823243482103402,
+      "eval_e3_heavy_atoms_difference": 0.5926646045072912,
+      "eval_e3_heavy_atoms_difference_norm": 0.014343698427793726,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9823243482103402,
+      "eval_has_all_attachment_points": 0.9860362350861688,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 9.245249668581529,
+      "eval_heavy_atoms_difference_norm": 0.12076758137682525,
+      "eval_linker_equal": 0.7355722492266903,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9948740609809986,
+      "eval_linker_heavy_atoms_difference": 0.6543526292532037,
+      "eval_linker_heavy_atoms_difference_norm": 0.021436092914249572,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9948740609809986,
+      "eval_loss": 0.282554566860199,
+      "eval_num_fragments": 2.999381352187362,
+      "eval_poi_equal": 0.7440565620857269,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9083517454706143,
+      "eval_poi_heavy_atoms_difference": 2.9160406539991164,
+      "eval_poi_heavy_atoms_difference_norm": 0.09447484708417836,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9083517454706143,
+      "eval_reassembly": 0.5122403888643394,
+      "eval_reassembly_nostereo": 0.5361908970393283,
+      "eval_runtime": 1510.4782,
+      "eval_samples_per_second": 7.491,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8894387980556783,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.05843953415751457,
+      "learning_rate": 2.8579230188804133e-06,
+      "loss": 0.0055,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.15142107009887695,
+      "learning_rate": 0.00015748506394475155,
+      "loss": 0.003,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.045971889048814774,
+      "learning_rate": 7.177185293925321e-07,
+      "loss": 0.0059,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.11659257858991623,
+      "learning_rate": 0.00015891725956903888,
+      "loss": 0.0029,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.0705426037311554,
+      "learning_rate": 0.0,
+      "loss": 0.0056,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5702165267344234,
+      "eval_e3_equal": 0.8179407865665046,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.985505965532479,
+      "eval_e3_heavy_atoms_difference": 0.5108263367211666,
+      "eval_e3_heavy_atoms_difference_norm": 0.010294462933377614,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.985505965532479,
+      "eval_has_all_attachment_points": 0.9895713654441007,
+      "eval_has_three_substructures": 0.9994697304463102,
+      "eval_heavy_atoms_difference": 7.444012372956252,
+      "eval_heavy_atoms_difference_norm": 0.09784689114716345,
+      "eval_linker_equal": 0.80830755634114,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9969951391957579,
+      "eval_linker_heavy_atoms_difference": 0.3240830755634114,
+      "eval_linker_heavy_atoms_difference_norm": 0.0026034531075389516,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9969951391957579,
+      "eval_loss": 0.28070321679115295,
+      "eval_num_fragments": 3.0,
+      "eval_poi_equal": 0.7792311091471498,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9240830755634114,
+      "eval_poi_heavy_atoms_difference": 2.270614228899691,
+      "eval_poi_heavy_atoms_difference_norm": 0.07370347784321037,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9240830755634114,
+      "eval_reassembly": 0.578965974370305,
+      "eval_reassembly_nostereo": 0.6044189129474149,
+      "eval_runtime": 1509.1643,
+      "eval_samples_per_second": 7.498,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9090587715422006,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=6-learning_rate=1.6e-04-warmup_ratio=0.010-num_cycles=9.500",
   "trial_params": {