Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f9bd8fdf0af32af8f59abce260f9b57d6b83c268c98baf68e11cc08f06c96e2
+oid sha256:953144d57e4c3be2b1c3d4615aeaeaffa4604e84762c3794c89daeb74095bce6
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:489808c2860cdf751eb2c99475a6b346765860ebbe7e08073938bc45654ce999
+oid sha256:224c67a4ba82ee6a14c66f2d316c876cb190e47426b2f49938c848909bc096ef
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc8958737f63827fcb5b01c7daddc2f1f0e348e84c3608ec0d26115852593ed2
+oid sha256:99e514efd4ffa03f15a6bea0655d7a9407689b39ce733a792383f24bd61549fe
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88aeefb87e8a5cdeb363b3bb85c7ea47c59528a3b28554e6fd533407ab39a79a
+oid sha256:06df29d56e01b453830a17398c04fe47fdf9b99a122e13f189ae42a3e719477b
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.07273530711444985,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=6/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.1444038152694702,
-      "learning_rate": 8.921312073289648e-06,
-      "loss": 2.3977,
+      "grad_norm": 0.3589436709880829,
+      "learning_rate": 0.00037653255646528873,
+      "loss": 0.8727,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.1081920862197876,
-      "learning_rate": 8.346964207338686e-06,
-      "loss": 0.6966,
+      "grad_norm": 0.31444501876831055,
+      "learning_rate": 0.0001876216539684001,
+      "loss": 0.1862,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.9350355267524719,
-      "learning_rate": 7.391792877870348e-06,
-      "loss": 0.4502,
+      "grad_norm": 0.40511536598205566,
+      "learning_rate": 2.4455100751933555e-05,
+      "loss": 0.1653,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.7580628395080566,
-      "learning_rate": 6.151179358577131e-06,
-      "loss": 0.3328,
+      "grad_norm": 0.24769523739814758,
+      "learning_rate": 0.00041247345429193725,
+      "loss": 0.174,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.879012405872345,
-      "learning_rate": 4.749008542126604e-06,
-      "loss": 0.2676,
+      "grad_norm": 0.25206995010375977,
+      "learning_rate": 0.0002497487544510346,
+      "loss": 0.1702,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.7280972003936768,
-      "learning_rate": 3.3252980717220435e-06,
-      "loss": 0.2305,
+      "grad_norm": 0.3927709758281708,
+      "learning_rate": 6.0588710738437684e-05,
+      "loss": 0.1575,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.47845959663391113,
-      "learning_rate": 2.022216492259077e-06,
-      "loss": 0.2078,
+      "grad_norm": 0.25023865699768066,
+      "learning_rate": 0.00043268145449372675,
+      "loss": 0.1532,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.5540239810943604,
-      "learning_rate": 9.698866172476045e-07,
-      "loss": 0.1961,
+      "grad_norm": 0.22349177300930023,
+      "learning_rate": 0.0003093259447288672,
+      "loss": 0.1749,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.6173533201217651,
-      "learning_rate": 2.733917555418904e-07,
-      "loss": 0.189,
+      "grad_norm": 0.3404127359390259,
+      "learning_rate": 0.00010949703228363757,
+      "loss": 0.1554,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.6460208296775818,
-      "learning_rate": 2.28232703021863e-09,
-      "loss": 0.1878,
+      "grad_norm": 0.3514672815799713,
+      "learning_rate": 1.0282592565903957e-06,
+      "loss": 0.1492,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.001237295625276182,
-      "eval_e3_equal": 0.2914714980114892,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.005391073795846222,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.993813521873619,
-      "eval_e3_heavy_atoms_difference": -1.2530269553689792,
-      "eval_e3_heavy_atoms_difference_norm": -0.06816270339643937,
+      "eval_e3_has_attachment_point(s)": 1.0,
+      "eval_e3_heavy_atoms_difference": -7.713742819266461,
+      "eval_e3_heavy_atoms_difference_norm": -0.4003938464081166,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.993813521873619,
-      "eval_has_all_attachment_points": 0.94025629695095,
-      "eval_has_three_substructures": 0.9953159522757402,
-      "eval_heavy_atoms_difference": 23.324701723376048,
-      "eval_heavy_atoms_difference_norm": 0.3003596358168922,
-      "eval_linker_equal": 0.00901458241272647,
+      "eval_e3_valid": 1.0,
+      "eval_has_all_attachment_points": 1.0,
+      "eval_has_three_substructures": 1.0,
+      "eval_heavy_atoms_difference": -26.404330534688466,
+      "eval_heavy_atoms_difference_norm": -0.4024205454430223,
+      "eval_linker_equal": 0.0011489173663278833,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9865665046398586,
-      "eval_linker_heavy_atoms_difference": 5.095183384887318,
-      "eval_linker_heavy_atoms_difference_norm": 0.21271914204862793,
+      "eval_linker_has_attachment_point(s)": 1.0,
+      "eval_linker_heavy_atoms_difference": -3.474856385329209,
+      "eval_linker_heavy_atoms_difference_norm": -0.49821316014574196,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9865665046398586,
-      "eval_loss": 0.3655618727207184,
-      "eval_num_fragments": 2.997613787008396,
-      "eval_poi_equal": 0.2577110030932391,
+      "eval_linker_valid": 1.0,
+      "eval_loss": 0.49772489070892334,
+      "eval_num_fragments": 3.0,
+      "eval_poi_equal": 0.001237295625276182,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.7584622182942996,
-      "eval_poi_heavy_atoms_difference": 8.438621299160406,
-      "eval_poi_heavy_atoms_difference_norm": 0.24540271262613053,
+      "eval_poi_has_attachment_point(s)": 1.0,
+      "eval_poi_heavy_atoms_difference": -15.215731330092797,
+      "eval_poi_heavy_atoms_difference_norm": -0.706030845323288,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.7584622182942996,
-      "eval_reassembly": 0.001237295625276182,
-      "eval_reassembly_nostereo": 0.0021210782147591694,
-      "eval_runtime": 1537.2247,
-      "eval_samples_per_second": 7.361,
-      "eval_steps_per_second": 0.115,
+      "eval_poi_valid": 1.0,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 1750.4647,
+      "eval_samples_per_second": 6.464,
+      "eval_steps_per_second": 0.101,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.7521873619089704,
+      "eval_valid": 1.0,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.47060030698776245,
-      "learning_rate": 8.882979678601097e-06,
-      "loss": 0.174,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.5330048203468323,
-      "learning_rate": 8.267282522223007e-06,
-      "loss": 0.1418,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.4379560947418213,
-      "learning_rate": 7.278718737276227e-06,
-      "loss": 0.1199,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.4775562286376953,
-      "learning_rate": 6.016004093409107e-06,
-      "loss": 0.1041,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.38391777873039246,
-      "learning_rate": 4.605230452182494e-06,
-      "loss": 0.0923,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.03853292090145824,
-      "eval_e3_equal": 0.54909412284578,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9959346000883783,
-      "eval_e3_heavy_atoms_difference": -0.3578435704816615,
-      "eval_e3_heavy_atoms_difference_norm": -0.02785724707369664,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9959346000883783,
-      "eval_has_all_attachment_points": 0.9732213875386655,
-      "eval_has_three_substructures": 0.997613787008396,
-      "eval_heavy_atoms_difference": 11.28157313300928,
-      "eval_heavy_atoms_difference_norm": 0.1489927886833175,
-      "eval_linker_equal": 0.08395934600088378,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9931064958020327,
-      "eval_linker_heavy_atoms_difference": 2.419001325673884,
-      "eval_linker_heavy_atoms_difference_norm": 0.08873987335070188,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9931064958020327,
-      "eval_loss": 0.3142109215259552,
-      "eval_num_fragments": 2.9990278391515686,
-      "eval_poi_equal": 0.5286787450287229,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8821917808219178,
-      "eval_poi_heavy_atoms_difference": 3.70234202386213,
-      "eval_poi_heavy_atoms_difference_norm": 0.1184160547254355,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8821917808219178,
-      "eval_reassembly": 0.03959346000883782,
-      "eval_reassembly_nostereo": 0.04958020326999558,
-      "eval_runtime": 1522.9606,
-      "eval_samples_per_second": 7.43,
-      "eval_steps_per_second": 0.116,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8780380026513478,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.3485475778579712,
-      "learning_rate": 3.187274515650354e-06,
-      "loss": 0.0844,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.31034043431282043,
-      "learning_rate": 1.9037301938199667e-06,
-      "loss": 0.0789,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.427944153547287,
-      "learning_rate": 8.827693533292254e-07,
-      "loss": 0.0752,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.35361239314079285,
-      "learning_rate": 2.2634286137006849e-07,
-      "loss": 0.0738,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.42388033866882324,
-      "learning_rate": 0.0,
-      "loss": 0.0731,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.07273530711444985,
-      "eval_e3_equal": 0.5886875828546178,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9959346000883783,
-      "eval_e3_heavy_atoms_difference": -0.4637207247017234,
-      "eval_e3_heavy_atoms_difference_norm": -0.03239218121458508,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9959346000883783,
-      "eval_has_all_attachment_points": 0.9784357048166151,
-      "eval_has_three_substructures": 0.9978789217852408,
-      "eval_heavy_atoms_difference": 9.685461776403004,
-      "eval_heavy_atoms_difference_norm": 0.1285929207163922,
-      "eval_linker_equal": 0.13504197967300044,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9935483870967742,
-      "eval_linker_heavy_atoms_difference": 1.6614228899690675,
-      "eval_linker_heavy_atoms_difference_norm": 0.048918556931556306,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9935483870967742,
-      "eval_loss": 0.30737239122390747,
-      "eval_num_fragments": 2.999116217410517,
-      "eval_poi_equal": 0.573663278833407,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8909412284577993,
-      "eval_poi_heavy_atoms_difference": 3.2883782589482986,
-      "eval_poi_heavy_atoms_difference_norm": 0.10593427106256059,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8909412284577993,
-      "eval_reassembly": 0.07476800707026071,
-      "eval_reassembly_nostereo": 0.0899690676093681,
-      "eval_runtime": 1508.2236,
-      "eval_samples_per_second": 7.502,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8865223155103844,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2",
+  "trial_name": "trial-number=10-learning_rate=4.4e-04-warmup_ratio=0.020-num_cycles=6",
   "trial_params": {
-    "learning_rate": 9.066610391784642e-06,
-    "num_cycles": 2,
-    "warmup_ratio": 0.060000000000000005
+    "learning_rate": 0.00043654768433250334,
+    "num_cycles": 6,
+    "warmup_ratio": 0.02
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f4dd62486422c15e6a55bb71f34dd6dc931136ad9b366ebdee4dcb75d7c3cb7
+oid sha256:bff5efeef8db362107d1e63facdaaf129ec43c620a9ef8b536b99025c233aad9
 size 7480