Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a73cb68c86cbe1848635e42d7b4d6f9390303de065c805e2870e459ca082b76f
+oid sha256:8d59053eb1d4b18159b41cd956b8b4aac56bc8b02e179b3c8bee9fe67e3eb2d1
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f745250fcb395bdec08e4f52aeea781643572fabe5450591968beb77f346e6e9
+oid sha256:c898935023b5524671ead799321d0cd4851a31cd0fc15a39d2fd22db27efafcb
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:304501a6e20e28711654c9cab7dda6bd9c1bb171d5282a43c8c3270133e6030f
+oid sha256:c6e0d2981c76f4dd5c860fba27bb2807616de6d455d4a9897613478c16859c39
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba354c2f296f218a940cdf4910b60996a290ca1f9d26acfeb6e3efddabc0500d
+oid sha256:49fe2533fd018c11c4612bd197ff5a3802e29314d75e117398a06c9b66f2971d
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=6/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5189571365444101,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=11-learning_rate=1.0e-04-warmup_ratio=0.080-num_cycles=3/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.3589436709880829,
-      "learning_rate": 0.00037653255646528873,
-      "loss": 0.8727,
+      "grad_norm": 0.6575852632522583,
+      "learning_rate": 0.000100182631515627,
+      "loss": 1.1484,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.31444501876831055,
-      "learning_rate": 0.0001876216539684001,
-      "loss": 0.1862,
+      "grad_norm": 0.343759149312973,
+      "learning_rate": 8.597337690594066e-05,
+      "loss": 0.1449,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.40511536598205566,
-      "learning_rate": 2.4455100751933555e-05,
-      "loss": 0.1653,
+      "grad_norm": 0.2601257562637329,
+      "learning_rate": 6.420018709015348e-05,
+      "loss": 0.0439,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.24769523739814758,
-      "learning_rate": 0.00041247345429193725,
-      "loss": 0.174,
+      "grad_norm": 0.13981187343597412,
+      "learning_rate": 3.9703875822711815e-05,
+      "loss": 0.0221,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.25206995010375977,
-      "learning_rate": 0.0002497487544510346,
-      "loss": 0.1702,
+      "grad_norm": 0.1797427237033844,
+      "learning_rate": 1.793068600692463e-05,
+      "loss": 0.0141,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.3927709758281708,
-      "learning_rate": 6.0588710738437684e-05,
-      "loss": 0.1575,
+      "grad_norm": 0.13262967765331268,
+      "learning_rate": 3.721431397238288e-06,
+      "loss": 0.011,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.25023865699768066,
-      "learning_rate": 0.00043268145449372675,
-      "loss": 0.1532,
+      "grad_norm": 0.24220901727676392,
+      "learning_rate": 0.00010366882009194501,
+      "loss": 0.0124,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.22349177300930023,
-      "learning_rate": 0.0003093259447288672,
-      "loss": 0.1749,
+      "grad_norm": 0.15627698600292206,
+      "learning_rate": 9.565686145691327e-05,
+      "loss": 0.0201,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.3404127359390259,
-      "learning_rate": 0.00010949703228363757,
-      "loss": 0.1554,
+      "grad_norm": 0.12300624698400497,
+      "learning_rate": 7.792804718464899e-05,
+      "loss": 0.0093,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.3514672815799713,
-      "learning_rate": 1.0282592565903957e-06,
-      "loss": 0.1492,
+      "grad_norm": 0.09846580028533936,
+      "learning_rate": 5.442400864406504e-05,
+      "loss": 0.006,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.005391073795846222,
+      "eval_all_ligands_equal": 0.5189571365444101,
+      "eval_e3_equal": 0.7959346000883782,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -7.713742819266461,
-      "eval_e3_heavy_atoms_difference_norm": -0.4003938464081166,
+      "eval_e3_has_attachment_point(s)": 0.9897481219619974,
+      "eval_e3_heavy_atoms_difference": 0.609191338930623,
+      "eval_e3_heavy_atoms_difference_norm": 0.013221863982469433,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -26.404330534688466,
-      "eval_heavy_atoms_difference_norm": -0.4024205454430223,
-      "eval_linker_equal": 0.0011489173663278833,
+      "eval_e3_valid": 0.9897481219619974,
+      "eval_has_all_attachment_points": 0.987273530711445,
+      "eval_has_three_substructures": 0.9997348652231551,
+      "eval_heavy_atoms_difference": 3.68519664162616,
+      "eval_heavy_atoms_difference_norm": 0.047260073323281075,
+      "eval_linker_equal": 0.7496243923994698,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -3.474856385329209,
-      "eval_linker_heavy_atoms_difference_norm": -0.49821316014574196,
+      "eval_linker_has_attachment_point(s)": 0.9977021652673442,
+      "eval_linker_heavy_atoms_difference": 0.32470172337604947,
+      "eval_linker_heavy_atoms_difference_norm": -0.001965493580026269,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.49772489070892334,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.001237295625276182,
+      "eval_linker_valid": 0.9977021652673442,
+      "eval_loss": 0.3125883936882019,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.7557224922669024,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -15.215731330092797,
-      "eval_poi_heavy_atoms_difference_norm": -0.706030845323288,
+      "eval_poi_has_attachment_point(s)": 0.9706584180291649,
+      "eval_poi_heavy_atoms_difference": 0.7122403888643394,
+      "eval_poi_heavy_atoms_difference_norm": 0.010501165767226064,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1750.4647,
-      "eval_samples_per_second": 6.464,
-      "eval_steps_per_second": 0.101,
+      "eval_poi_valid": 0.9706584180291649,
+      "eval_reassembly": 0.5268228015908086,
+      "eval_reassembly_nostereo": 0.552364118426867,
+      "eval_runtime": 1535.8187,
+      "eval_samples_per_second": 7.367,
+      "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
+      "eval_valid": 0.9590808661069377,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.22048687934875488,
-      "learning_rate": 0.0003615266351647806,
-      "loss": 0.1815,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.29270756244659424,
-      "learning_rate": 0.00016721780404492913,
-      "loss": 0.1562,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.3407532870769501,
-      "learning_rate": 1.5790233792515393e-05,
-      "loss": 0.1492,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 1.7223341464996338,
-      "learning_rate": 0.0004021218362691176,
-      "loss": 0.1594,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.24435298144817352,
-      "learning_rate": 0.00022907483237285996,
-      "loss": 0.1596,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.009721608484312859,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.8992487847989394,
-      "eval_e3_heavy_atoms_difference": 5.7851524524966855,
-      "eval_e3_heavy_atoms_difference_norm": 0.12661446588878553,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.8992487847989394,
-      "eval_has_all_attachment_points": 0.8992487847989394,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 1.4468404772425982,
-      "eval_heavy_atoms_difference_norm": -0.019586192312177807,
-      "eval_linker_equal": 0.001855943437914273,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -0.8486080424215643,
-      "eval_linker_heavy_atoms_difference_norm": -0.29747536554098253,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.47853395342826843,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0015024304021210783,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -8.162881131241715,
-      "eval_poi_heavy_atoms_difference_norm": -0.44042194138209373,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1424.7796,
-      "eval_samples_per_second": 7.942,
-      "eval_steps_per_second": 0.124,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8992487847989394,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2928465008735657,
-      "learning_rate": 4.695622606249373e-05,
-      "loss": 0.1489,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.3326316177845001,
-      "learning_rate": 0.0004278227662773617,
-      "loss": 0.1491,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.20325560867786407,
-      "learning_rate": 0.00029005682871021413,
-      "loss": 0.1648,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.30880847573280334,
-      "learning_rate": 9.200135272410474e-05,
-      "loss": 0.1494,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.3054123818874359,
-      "learning_rate": 3.9564294260980494e-10,
-      "loss": 0.1455,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.006628369421122404,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -4.341847105612019,
-      "eval_e3_heavy_atoms_difference_norm": -0.2680214691576094,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -13.891294741493592,
-      "eval_heavy_atoms_difference_norm": -0.22988441728596545,
-      "eval_linker_equal": 0.0016791869200176756,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9983208130799823,
-      "eval_linker_heavy_atoms_difference": -2.50128148475475,
-      "eval_linker_heavy_atoms_difference_norm": -0.42292556753429383,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9983208130799823,
-      "eval_loss": 0.4854263961315155,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.001590808661069377,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9999116217410517,
-      "eval_poi_heavy_atoms_difference": -7.16880247459125,
-      "eval_poi_heavy_atoms_difference_norm": -0.4058157521929436,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9999116217410517,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1622.9767,
-      "eval_samples_per_second": 6.972,
-      "eval_steps_per_second": 0.109,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9982324348210341,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=6",
+  "trial_name": "trial-number=11-learning_rate=1.0e-04-warmup_ratio=0.080-num_cycles=3",
   "trial_params": {
-    "learning_rate": 0.00043654768433250334,
-    "num_cycles": 6,
-    "warmup_ratio": 0.02
+    "learning_rate": 0.00010390406291286528,
+    "num_cycles": 3,
+    "warmup_ratio": 0.08
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bff5efeef8db362107d1e63facdaaf129ec43c620a9ef8b536b99025c233aad9
+oid sha256:79e8dd2f501b00cb907081399791da6f3e67acafeecf3f0d96b43e4e21e6e9c2
 size 7480