Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bf9c6ea4115da7ecb26efa2880f84d13e203883e1744e63861c53658f9277f2
+oid sha256:1982171a3f3820e5b58f6001600cc37642dc0186945f9750c194a1fa5b77dc7b
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b6e2d89d31809aafc6d9f1a3d4b804a258907c5f3f1057e64ab7c0b2b18e13b
+oid sha256:df855786c8c1dcf532c502a2b833691d5e70e66943335a077622d762846c4f43
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e864df1cdce41b5f6008f2f3056157fe1a59741e45538c35750dad335d89adcf
+oid sha256:4e7771a01cd6f59c5ffb87e70f444df0eea884e2c0b2aaf819bce5c2c8e7a461
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b2e2f4a4d698b32d29e4f198023bd7f534f0e63616c12fe3bd5d6eef023d892
+oid sha256:93005377f6086337612335e1f5fcb47fe8e2d6ecd39daaae4c9fed9a1602b473
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.16827220503756077,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.038974812196199735,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=8.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.1925102472305298,
-      "learning_rate": 9.50852571041643e-06,
-      "loss": 3.5759,
+      "grad_norm": 1.1311782598495483,
+      "learning_rate": 1.4046924613685011e-05,
+      "loss": 3.1644,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.0559757947921753,
-      "learning_rate": 6.9505467525657206e-06,
-      "loss": 0.7964,
+      "grad_norm": 0.9366589784622192,
+      "learning_rate": 8.278052011897948e-06,
+      "loss": 0.6401,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.9585791230201721,
-      "learning_rate": 2.6025037504397807e-06,
-      "loss": 0.5237,
+      "grad_norm": 0.9480023980140686,
+      "learning_rate": 8.711793227257574e-06,
+      "loss": 0.4026,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.8004510998725891,
-      "learning_rate": 1.2979375408792664e-05,
-      "loss": 0.44,
+      "grad_norm": 0.6237757205963135,
+      "learning_rate": 1.3493023962384968e-05,
+      "loss": 0.3056,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.5827901363372803,
-      "learning_rate": 2.9673475846258797e-07,
-      "loss": 0.3022,
+      "grad_norm": 0.546779215335846,
+      "learning_rate": 3.844673201988863e-06,
+      "loss": 0.215,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.9724856615066528,
-      "learning_rate": 1.0734810793876912e-05,
-      "loss": 0.2719,
+      "grad_norm": 0.6261709332466125,
+      "learning_rate": 1.765819137898168e-05,
+      "loss": 0.1886,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.5034233331680298,
-      "learning_rate": 6.2863182132847955e-06,
-      "loss": 0.2177,
+      "grad_norm": 0.3576153814792633,
+      "learning_rate": 6.844894722497407e-07,
+      "loss": 0.1428,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.5209541320800781,
-      "learning_rate": 3.1491776008674324e-06,
-      "loss": 0.1838,
+      "grad_norm": 0.5195497870445251,
+      "learning_rate": 1.958347656200152e-05,
+      "loss": 0.1311,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.5501710772514343,
-      "learning_rate": 1.2746394296240856e-05,
-      "loss": 0.1695,
+      "grad_norm": 0.48624593019485474,
+      "learning_rate": 1.341743352364844e-07,
+      "loss": 0.1046,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.6129121780395508,
-      "learning_rate": 1.3243314463025088e-07,
-      "loss": 0.1406,
+      "grad_norm": 0.45008236169815063,
+      "learning_rate": 1.871878422530564e-05,
+      "loss": 0.0948,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.007247017233760495,
-      "eval_e3_equal": 0.39010163499779055,
+      "eval_all_ligands_equal": 0.038974812196199735,
+      "eval_e3_equal": 0.5149801148917367,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.997348652231551,
-      "eval_e3_heavy_atoms_difference": -1.0644277507733098,
-      "eval_e3_heavy_atoms_difference_norm": -0.05826937320242394,
+      "eval_e3_has_attachment_point(s)": 0.9974370304904994,
+      "eval_e3_heavy_atoms_difference": -0.5439681838267786,
+      "eval_e3_heavy_atoms_difference_norm": -0.03979103911128965,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.997348652231551,
-      "eval_has_all_attachment_points": 0.9711886875828546,
-      "eval_has_three_substructures": 0.9977021652673442,
-      "eval_heavy_atoms_difference": 11.813079982324348,
-      "eval_heavy_atoms_difference_norm": 0.1508348868839678,
-      "eval_linker_equal": 0.02615996464869642,
+      "eval_e3_valid": 0.9974370304904994,
+      "eval_has_all_attachment_points": 0.9802916482545294,
+      "eval_has_three_substructures": 0.9978789217852408,
+      "eval_heavy_atoms_difference": 9.873530711444985,
+      "eval_heavy_atoms_difference_norm": 0.12658016008251008,
+      "eval_linker_equal": 0.08758285461776404,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9928413610251878,
-      "eval_linker_heavy_atoms_difference": 2.933097657976138,
-      "eval_linker_heavy_atoms_difference_norm": 0.08887640414726028,
+      "eval_linker_has_attachment_point(s)": 0.9955810870525851,
+      "eval_linker_heavy_atoms_difference": 3.0841361025187806,
+      "eval_linker_heavy_atoms_difference_norm": 0.12312258066016835,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9928413610251878,
-      "eval_loss": 0.33352142572402954,
+      "eval_linker_valid": 0.9955810870525851,
+      "eval_loss": 0.31889262795448303,
       "eval_num_fragments": 2.9984091913389306,
-      "eval_poi_equal": 0.4074237737516571,
+      "eval_poi_equal": 0.5147149801148917,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8798055678303137,
-      "eval_poi_heavy_atoms_difference": 3.964825452938577,
-      "eval_poi_heavy_atoms_difference_norm": 0.11040993122450134,
+      "eval_poi_has_attachment_point(s)": 0.9095006628369421,
+      "eval_poi_heavy_atoms_difference": 3.1498011489173665,
+      "eval_poi_heavy_atoms_difference_norm": 0.09194667815914018,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8798055678303137,
-      "eval_reassembly": 0.00760053026955369,
-      "eval_reassembly_nostereo": 0.010251878038002652,
-      "eval_runtime": 1609.8438,
-      "eval_samples_per_second": 7.029,
-      "eval_steps_per_second": 0.11,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8760936809544851,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.5318701863288879,
-      "learning_rate": 1.1237446278275383e-05,
-      "loss": 0.1316,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.3978229761123657,
-      "learning_rate": 5.625686628642194e-06,
-      "loss": 0.1163,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.3678536117076874,
-      "learning_rate": 3.7307809795190627e-06,
-      "loss": 0.1024,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.4965149462223053,
-      "learning_rate": 1.2452489342403467e-05,
-      "loss": 0.0972,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.4630180895328522,
-      "learning_rate": 3.319116134821198e-08,
-      "loss": 0.0827,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.061511268228015906,
-      "eval_e3_equal": 0.5425541316836058,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9982324348210341,
-      "eval_e3_heavy_atoms_difference": -0.41617322138753865,
-      "eval_e3_heavy_atoms_difference_norm": -0.030251585246387123,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9982324348210341,
-      "eval_has_all_attachment_points": 0.9847105612019443,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 6.157048166151127,
-      "eval_heavy_atoms_difference_norm": 0.0799003116743603,
-      "eval_linker_equal": 0.12072470172337604,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9971718957136544,
-      "eval_linker_heavy_atoms_difference": 1.4752982766239504,
-      "eval_linker_heavy_atoms_difference_norm": 0.026323008455953905,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9971718957136544,
-      "eval_loss": 0.31108492612838745,
-      "eval_num_fragments": 2.999646486964207,
-      "eval_poi_equal": 0.5619973486522315,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9346884666372073,
-      "eval_poi_heavy_atoms_difference": 1.8780380026513477,
-      "eval_poi_heavy_atoms_difference_norm": 0.053725194951032486,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9346884666372073,
-      "eval_reassembly": 0.0625718073353955,
-      "eval_reassembly_nostereo": 0.07618205921343349,
-      "eval_runtime": 1600.5588,
-      "eval_samples_per_second": 7.069,
+      "eval_poi_valid": 0.9095006628369421,
+      "eval_reassembly": 0.04030048608042421,
+      "eval_reassembly_nostereo": 0.04904993371630579,
+      "eval_runtime": 1598.7498,
+      "eval_samples_per_second": 7.077,
       "eval_steps_per_second": 0.111,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9327441449403446,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.36397504806518555,
-      "learning_rate": 1.1694228723322586e-05,
-      "loss": 0.0784,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.3220004737377167,
-      "learning_rate": 4.975250136973397e-06,
-      "loss": 0.0712,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.33015140891075134,
-      "learning_rate": 4.341505051882401e-06,
-      "loss": 0.0627,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.3355898857116699,
-      "learning_rate": 1.2100595958768545e-05,
-      "loss": 0.0608,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.2892158329486847,
-      "learning_rate": 0.0,
-      "loss": 0.0528,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.16827220503756077,
-      "eval_e3_equal": 0.639063190455148,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9983208130799823,
-      "eval_e3_heavy_atoms_difference": -0.2734423331860362,
-      "eval_e3_heavy_atoms_difference_norm": -0.024892349019047463,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9983208130799823,
-      "eval_has_all_attachment_points": 0.9885992045956694,
-      "eval_has_three_substructures": 0.9992045956694653,
-      "eval_heavy_atoms_difference": 4.554838709677419,
-      "eval_heavy_atoms_difference_norm": 0.058807906961281145,
-      "eval_linker_equal": 0.28501988510826337,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9977905435262925,
-      "eval_linker_heavy_atoms_difference": 0.91064958020327,
-      "eval_linker_heavy_atoms_difference_norm": 0.007958336622586505,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9977905435262925,
-      "eval_loss": 0.30263492465019226,
-      "eval_num_fragments": 2.999734865223155,
-      "eval_poi_equal": 0.632523199292974,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9508616880247459,
-      "eval_poi_heavy_atoms_difference": 1.4015908086610693,
-      "eval_poi_heavy_atoms_difference_norm": 0.0383393486603852,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9508616880247459,
-      "eval_reassembly": 0.1715422006186478,
-      "eval_reassembly_nostereo": 0.1992045956694653,
-      "eval_runtime": 1518.0837,
-      "eval_samples_per_second": 7.453,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9489173663278834,
-      "step": 10000
+      "eval_valid": 0.9072912063632347,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500",
+  "trial_name": "trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=8.500",
   "trial_params": {
-    "learning_rate": 1.329291894316217e-05,
-    "num_cycles": 7.5,
-    "warmup_ratio": 0.09999999999999999
+    "learning_rate": 1.9637600609931647e-05,
+    "num_cycles": 8.5,
+    "warmup_ratio": 0.09
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e738cf844f012a143f915952b9b5d140ea4bbe7014578abe5deedc776149cbdd
+oid sha256:8b7a3dced9d5154a95ec1a0505e00dc47ab6d5dbc40b63df76ec9c233913bd4b
 size 7480