Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca4fe7c2b27306b825b74ed0fca6c007623edcc95ed6a1066831daad87297111
+oid sha256:55f59133b63fec3a4a556777b07d6262cf0d2bf767b8eefd4c4ea17ad9bf0ba3
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2453bf4e543ffc35925652d75fb01c1ae88dc37c9f16530e003f8bfc84e7993
+oid sha256:2adafed36bd362d33a3d922e35d1197bebba7f38a9b0df528254d0a1c319e58e
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:737c37e64d8976b6a98de2e50f10afefd5a40eb5d5622d8f238c62b443f8adc0
+oid sha256:ac54a773cfe92e4c2a29bf082fd59fe36794afd9085f9d462c8ff207b02fe922
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a0422bab0c4fbf63011708fabc615bc4cfb890b30611cd7933acaf26bce321d
+oid sha256:b9e2464715e1e57f2ca6ab165527ccdb3a163130fec9e8ac9e7e34655c5e25d9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.4693769332744145,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=3-learning_rate=5.4e-05-warmup_ratio=0.060-num_cycles=7.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5506849315068493,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=3-learning_rate=5.4e-05-warmup_ratio=0.060-num_cycles=7.500/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9180733539549271,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.1673220545053482,
+      "learning_rate": 4.5792422986113385e-05,
+      "loss": 0.0115,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.1533503383398056,
+      "learning_rate": 2.292458760707345e-05,
+      "loss": 0.0133,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.11694014817476273,
+      "learning_rate": 1.5202875853828108e-05,
+      "loss": 0.0074,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.20582005381584167,
+      "learning_rate": 5.074370502663012e-05,
+      "loss": 0.0099,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.10524490475654602,
+      "learning_rate": 1.3525347861248427e-07,
+      "loss": 0.0077,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.525143614670791,
+      "eval_e3_equal": 0.8016791869200177,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9950508174988952,
+      "eval_e3_heavy_atoms_difference": 0.3802032699955811,
+      "eval_e3_heavy_atoms_difference_norm": 0.004701090074063982,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9950508174988952,
+      "eval_has_all_attachment_points": 0.9864781263809103,
+      "eval_has_three_substructures": 0.9994697304463102,
+      "eval_heavy_atoms_difference": 6.110207688908528,
+      "eval_heavy_atoms_difference_norm": 0.08156866603906228,
+      "eval_linker_equal": 0.747326557666814,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9962881131241714,
+      "eval_linker_heavy_atoms_difference": 0.3719840919133893,
+      "eval_linker_heavy_atoms_difference_norm": 0.0025408818504140404,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9962881131241714,
+      "eval_loss": 0.2915845215320587,
+      "eval_num_fragments": 3.0001767565178965,
+      "eval_poi_equal": 0.7601414052143173,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9361908970393283,
+      "eval_poi_heavy_atoms_difference": 1.8801590808661068,
+      "eval_poi_heavy_atoms_difference_norm": 0.06152053159377357,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9361908970393283,
+      "eval_reassembly": 0.5324790101634997,
+      "eval_reassembly_nostereo": 0.5604065399911622,
+      "eval_runtime": 1513.416,
+      "eval_samples_per_second": 7.476,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9293857711003093,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.1497071534395218,
+      "learning_rate": 4.765380451517755e-05,
+      "loss": 0.0056,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.1030835211277008,
+      "learning_rate": 2.027406877792606e-05,
+      "loss": 0.0081,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.10188595950603485,
+      "learning_rate": 1.769156717718772e-05,
+      "loss": 0.004,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.16282129287719727,
+      "learning_rate": 4.930974483047975e-05,
+      "loss": 0.0059,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.10255538672208786,
+      "learning_rate": 0.0,
+      "loss": 0.0045,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5506849315068493,
+      "eval_e3_equal": 0.8073353954927088,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9875386654882898,
+      "eval_e3_heavy_atoms_difference": 0.5570481661511268,
+      "eval_e3_heavy_atoms_difference_norm": 0.011100557286704284,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9875386654882898,
+      "eval_has_all_attachment_points": 0.9839151568714096,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 6.227927529827662,
+      "eval_heavy_atoms_difference_norm": 0.08341091302743117,
+      "eval_linker_equal": 0.7790543526292532,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9955810870525851,
+      "eval_linker_heavy_atoms_difference": 0.29359257622624835,
+      "eval_linker_heavy_atoms_difference_norm": -0.0008758357451582713,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9955810870525851,
+      "eval_loss": 0.3028588891029358,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.7679186920017675,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9393725143614671,
+      "eval_poi_heavy_atoms_difference": 1.769774635439682,
+      "eval_poi_heavy_atoms_difference_norm": 0.05719721998360895,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9393725143614671,
+      "eval_reassembly": 0.5586389748121962,
+      "eval_reassembly_nostereo": 0.5856827220503756,
+      "eval_runtime": 1514.7229,
+      "eval_samples_per_second": 7.47,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9255855059655325,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=3-learning_rate=5.4e-05-warmup_ratio=0.060-num_cycles=7.500",
   "trial_params": {