Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dafdcd076174d645b89b963fd7508d233988b5151b8856e96648280a9ddd6439
+oid sha256:8fb45637fb65c35af8768a36b33d1c9681556a3e01da28c8ecc31868a401b89d
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2956a5d075d4ebd9211b11b394dc7111af480ae699847d04a12f721f693c50e7
+oid sha256:a192a262b1834f01bca9dad7258fe2d1c4eb780ccbcc5369777aed8e3c2fe542
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e53b24e9a5fc9d7086dabf7ea15ea8545ea4c691e97ca71794604ae7dd9e671
+oid sha256:38fea94f5a2552b54264ba37d51c5e23e8f780a43c0b6021815a435ce42bab4c
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db54f45be63f2a31cb673e60b81aae6b0e3b4b54ab7baae14cb41e91340d7eb8
+oid sha256:3fda06869ae896845cef697d74c9284a49d9f0497862670a0e60cb59f13c3372
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.4258948298718515,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5652673442333186,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8826336721166593,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.0855363979935646,
+      "learning_rate": 2.739995113403538e-05,
+      "loss": 0.0135,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.07733646780252457,
+      "learning_rate": 0.00017844822299033737,
+      "loss": 0.0045,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.08803705126047134,
+      "learning_rate": 7.810415658232181e-05,
+      "loss": 0.011,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.07920731604099274,
+      "learning_rate": 0.00012044362176156312,
+      "loss": 0.0037,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.09854772686958313,
+      "learning_rate": 0.00013983023944795432,
+      "loss": 0.0175,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.5083517454706142,
+      "eval_e3_equal": 0.8068935041979673,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9902783915156872,
+      "eval_e3_heavy_atoms_difference": 0.5395492708793637,
+      "eval_e3_heavy_atoms_difference_norm": 0.01319573072161701,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9902783915156872,
+      "eval_has_all_attachment_points": 0.9885992045956694,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 8.121785240830755,
+      "eval_heavy_atoms_difference_norm": 0.10144213714960434,
+      "eval_linker_equal": 0.7551038444542643,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9975254087494476,
+      "eval_linker_heavy_atoms_difference": 0.45152452496685813,
+      "eval_linker_heavy_atoms_difference_norm": 0.007385939888565608,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9975254087494476,
+      "eval_loss": 0.28004351258277893,
+      "eval_num_fragments": 2.999381352187362,
+      "eval_poi_equal": 0.7333627927529828,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9210782147591693,
+      "eval_poi_heavy_atoms_difference": 3.063985859478568,
+      "eval_poi_heavy_atoms_difference_norm": 0.08532202235757833,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9210782147591693,
+      "eval_reassembly": 0.5161290322580645,
+      "eval_reassembly_nostereo": 0.544675209898365,
+      "eval_runtime": 1640.7276,
+      "eval_samples_per_second": 6.896,
+      "eval_steps_per_second": 0.108,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9114449845338047,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.06846427172422409,
+      "learning_rate": 6.014314326583964e-05,
+      "loss": 0.0043,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.12191401422023773,
+      "learning_rate": 0.0001936769028943003,
+      "loss": 0.0067,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.043216489255428314,
+      "learning_rate": 1.601154649200009e-05,
+      "loss": 0.0044,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.17743472754955292,
+      "learning_rate": 0.00022488269882462762,
+      "loss": 0.0057,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.05701812356710434,
+      "learning_rate": 1.8865539543912193e-09,
+      "loss": 0.008,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5652673442333186,
+      "eval_e3_equal": 0.8191780821917808,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9943437914273089,
+      "eval_e3_heavy_atoms_difference": 0.23968183826778613,
+      "eval_e3_heavy_atoms_difference_norm": 0.0009516451755027366,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9943437914273089,
+      "eval_has_all_attachment_points": 0.9915156871409633,
+      "eval_has_three_substructures": 0.9999116217410517,
+      "eval_heavy_atoms_difference": 4.139372514361467,
+      "eval_heavy_atoms_difference_norm": 0.0527094987954026,
+      "eval_linker_equal": 0.8091029606716748,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9978789217852408,
+      "eval_linker_heavy_atoms_difference": 0.29960229783473263,
+      "eval_linker_heavy_atoms_difference_norm": -7.111423721457522e-05,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9978789217852408,
+      "eval_loss": 0.2858957052230835,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.7730446310207689,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9616438356164384,
+      "eval_poi_heavy_atoms_difference": 1.3501546619531595,
+      "eval_poi_heavy_atoms_difference_norm": 0.03637356926643038,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9616438356164384,
+      "eval_reassembly": 0.5741935483870968,
+      "eval_reassembly_nostereo": 0.6089262041537782,
+      "eval_runtime": 1532.3845,
+      "eval_samples_per_second": 7.384,
+      "eval_steps_per_second": 0.116,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.955015466195316,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.364959997072179e+16,
+  "total_flos": 8.729673191023411e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500",
   "trial_params": {