Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79fc0c976a52b4d035d62e321fcc6810b9be45416051054697ff1ef595b5d7bd
+oid sha256:c8f3df3263b291f011acb1bd7eae9562d95656086388348957e3fcf69a3b3fc8
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea6e75d178075cad121b58972b5e1c177a29b64527250e2698d782c2caeb622a
+oid sha256:c493ed86c017441629a294afd3e402aa5df8841767b0c675f33e055a83d226a9
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f3b19699ff70854e5b91e5083739cc00e14b863fd7331afcf19f6956b68c760
+oid sha256:c37095416e836aa443a98103580053b06879daab2d3b2c6a992cb43f1523aa0e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b2e2f4a4d698b32d29e4f198023bd7f534f0e63616c12fe3bd5d6eef023d892
+oid sha256:93005377f6086337612335e1f5fcb47fe8e2d6ecd39daaae4c9fed9a1602b473
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.21599646486964208,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.04922669023420238,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=8.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.178853988647461,
-      "learning_rate": 9.50852571041643e-06,
-      "loss": 3.581,
+      "grad_norm": 1.1961781978607178,
+      "learning_rate": 1.4046924613685011e-05,
+      "loss": 3.1671,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.0605878829956055,
-      "learning_rate": 6.9505467525657206e-06,
-      "loss": 0.7934,
+      "grad_norm": 0.884679913520813,
+      "learning_rate": 8.278052011897948e-06,
+      "loss": 0.6333,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.9815279245376587,
-      "learning_rate": 2.6025037504397807e-06,
-      "loss": 0.5172,
+      "grad_norm": 0.87437903881073,
+      "learning_rate": 8.711793227257574e-06,
+      "loss": 0.3929,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.8215715289115906,
-      "learning_rate": 1.2979375408792664e-05,
-      "loss": 0.4313,
+      "grad_norm": 0.6494759917259216,
+      "learning_rate": 1.3493023962384968e-05,
+      "loss": 0.2939,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.6357488632202148,
-      "learning_rate": 2.9673475846258797e-07,
-      "loss": 0.291,
+      "grad_norm": 0.5190607309341431,
+      "learning_rate": 3.844673201988863e-06,
+      "loss": 0.2022,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.8166929483413696,
-      "learning_rate": 1.0734810793876912e-05,
-      "loss": 0.2605,
+      "grad_norm": 0.5579734444618225,
+      "learning_rate": 1.765819137898168e-05,
+      "loss": 0.1755,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.49792003631591797,
-      "learning_rate": 6.2863182132847955e-06,
-      "loss": 0.206,
+      "grad_norm": 0.37152209877967834,
+      "learning_rate": 6.844894722497407e-07,
+      "loss": 0.1299,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.4976484477519989,
-      "learning_rate": 3.1491776008674324e-06,
-      "loss": 0.1716,
+      "grad_norm": 0.47187724709510803,
+      "learning_rate": 1.958347656200152e-05,
+      "loss": 0.118,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.6568047404289246,
-      "learning_rate": 1.2746394296240856e-05,
-      "loss": 0.1567,
+      "grad_norm": 0.49866971373558044,
+      "learning_rate": 1.341743352364844e-07,
+      "loss": 0.091,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.609093189239502,
-      "learning_rate": 1.3243314463025088e-07,
-      "loss": 0.1276,
+      "grad_norm": 0.460344523191452,
+      "learning_rate": 1.871878422530564e-05,
+      "loss": 0.0816,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.01034025629695095,
-      "eval_e3_equal": 0.4445426425099426,
+      "eval_all_ligands_equal": 0.04922669023420238,
+      "eval_e3_equal": 0.5640300486080424,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9977021652673442,
-      "eval_e3_heavy_atoms_difference": -0.8713212549712771,
-      "eval_e3_heavy_atoms_difference_norm": -0.05159754947905971,
+      "eval_e3_has_attachment_point(s)": 0.9963764913831198,
+      "eval_e3_heavy_atoms_difference": -0.31904551480335835,
+      "eval_e3_heavy_atoms_difference_norm": -0.029566970024015815,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9977021652673442,
-      "eval_has_all_attachment_points": 0.9650022094564737,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 13.456915598762704,
-      "eval_heavy_atoms_difference_norm": 0.17429577152470357,
-      "eval_linker_equal": 0.031816173221387536,
+      "eval_e3_valid": 0.9963764913831198,
+      "eval_has_all_attachment_points": 0.9717189571365444,
+      "eval_has_three_substructures": 0.9980556783031375,
+      "eval_heavy_atoms_difference": 14.78338488731772,
+      "eval_heavy_atoms_difference_norm": 0.19121559027342688,
+      "eval_linker_equal": 0.10136986301369863,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9928413610251878,
-      "eval_linker_heavy_atoms_difference": 2.90181175430844,
-      "eval_linker_heavy_atoms_difference_norm": 0.10054857729675626,
+      "eval_linker_has_attachment_point(s)": 0.9921343349536014,
+      "eval_linker_heavy_atoms_difference": 4.2574458683163945,
+      "eval_linker_heavy_atoms_difference_norm": 0.2116283312283488,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9928413610251878,
-      "eval_loss": 0.32929131388664246,
-      "eval_num_fragments": 2.9992929739284135,
-      "eval_poi_equal": 0.4438356164383562,
+      "eval_linker_valid": 0.9921343349536014,
+      "eval_loss": 0.3197636604309082,
+      "eval_num_fragments": 2.998232434821034,
+      "eval_poi_equal": 0.5109147149801149,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8600088378258949,
-      "eval_poi_heavy_atoms_difference": 4.574900574458683,
-      "eval_poi_heavy_atoms_difference_norm": 0.13869595437595278,
+      "eval_poi_has_attachment_point(s)": 0.8669907202828104,
+      "eval_poi_heavy_atoms_difference": 4.766769774635439,
+      "eval_poi_heavy_atoms_difference_norm": 0.1480294319304517,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8600088378258949,
-      "eval_reassembly": 0.010693769332744145,
-      "eval_reassembly_nostereo": 0.014405656208572691,
-      "eval_runtime": 1609.9466,
-      "eval_samples_per_second": 7.028,
-      "eval_steps_per_second": 0.11,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8556783031374282,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.450788676738739,
-      "learning_rate": 1.1237446278275383e-05,
-      "loss": 0.1188,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.3590855002403259,
-      "learning_rate": 5.625686628642194e-06,
-      "loss": 0.1036,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.3720811903476715,
-      "learning_rate": 3.7307809795190627e-06,
-      "loss": 0.0895,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.3603754937648773,
-      "learning_rate": 1.2452489342403467e-05,
-      "loss": 0.0845,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.36783722043037415,
-      "learning_rate": 3.319116134821198e-08,
-      "loss": 0.0703,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.08784798939460893,
-      "eval_e3_equal": 0.6030048608042422,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9977905435262925,
-      "eval_e3_heavy_atoms_difference": -0.48855501546619534,
-      "eval_e3_heavy_atoms_difference_norm": -0.03403781512694932,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9977905435262925,
-      "eval_has_all_attachment_points": 0.980468404772426,
-      "eval_has_three_substructures": 0.9989394608926204,
-      "eval_heavy_atoms_difference": 8.604949182501105,
-      "eval_heavy_atoms_difference_norm": 0.11393370060428205,
-      "eval_linker_equal": 0.15740167918692002,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9966416261599647,
-      "eval_linker_heavy_atoms_difference": 1.3753424657534246,
-      "eval_linker_heavy_atoms_difference_norm": 0.028364947308648997,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9966416261599647,
-      "eval_loss": 0.30729973316192627,
-      "eval_num_fragments": 2.99946973044631,
-      "eval_poi_equal": 0.583738400353513,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9022536456031817,
-      "eval_poi_heavy_atoms_difference": 2.9832081307998233,
-      "eval_poi_heavy_atoms_difference_norm": 0.09467243955867818,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9022536456031817,
-      "eval_reassembly": 0.08988068935041979,
-      "eval_reassembly_nostereo": 0.10543526292532038,
-      "eval_runtime": 1567.0877,
-      "eval_samples_per_second": 7.22,
-      "eval_steps_per_second": 0.113,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.899425541316836,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.3629905879497528,
-      "learning_rate": 1.1694228723322586e-05,
-      "loss": 0.0661,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.29829272627830505,
-      "learning_rate": 4.975250136973397e-06,
-      "loss": 0.0589,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.3572481870651245,
-      "learning_rate": 4.341505051882401e-06,
-      "loss": 0.0509,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.35196781158447266,
-      "learning_rate": 1.2100595958768545e-05,
-      "loss": 0.0494,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.3531056046485901,
-      "learning_rate": 0.0,
-      "loss": 0.0421,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.21599646486964208,
-      "eval_e3_equal": 0.6945647370746796,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9948740609809986,
-      "eval_e3_heavy_atoms_difference": -0.22342023862129917,
-      "eval_e3_heavy_atoms_difference_norm": -0.021405765345262846,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9948740609809986,
-      "eval_has_all_attachment_points": 0.9802916482545294,
-      "eval_has_three_substructures": 0.9986743261157756,
-      "eval_heavy_atoms_difference": 7.292266902342024,
-      "eval_heavy_atoms_difference_norm": 0.0977349900472062,
-      "eval_linker_equal": 0.3343349536014141,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9961997348652232,
-      "eval_linker_heavy_atoms_difference": 0.8371188687582855,
-      "eval_linker_heavy_atoms_difference_norm": 0.011876128156630182,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9961997348652232,
-      "eval_loss": 0.2998192608356476,
-      "eval_num_fragments": 3.000265134776845,
-      "eval_poi_equal": 0.6514361467079098,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9148033583738401,
-      "eval_poi_heavy_atoms_difference": 2.523376049491825,
-      "eval_poi_heavy_atoms_difference_norm": 0.08047266034814943,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9148033583738401,
-      "eval_reassembly": 0.2201502430402121,
-      "eval_reassembly_nostereo": 0.2419796730004419,
-      "eval_runtime": 1501.9039,
-      "eval_samples_per_second": 7.534,
+      "eval_poi_valid": 0.8669907202828104,
+      "eval_reassembly": 0.05064074237737517,
+      "eval_reassembly_nostereo": 0.05885992045956694,
+      "eval_runtime": 1493.6802,
+      "eval_samples_per_second": 7.575,
       "eval_steps_per_second": 0.118,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9097657976137871,
-      "step": 10000
+      "eval_valid": 0.8614228899690676,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=0-learning_rate=1.3e-05-warmup_ratio=0.100-num_cycles=7.500",
+  "trial_name": "trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=8.500",
   "trial_params": {
-    "learning_rate": 1.329291894316217e-05,
-    "num_cycles": 7.5,
-    "warmup_ratio": 0.09999999999999999
+    "learning_rate": 1.9637600609931647e-05,
+    "num_cycles": 8.5,
+    "warmup_ratio": 0.09
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db6cdf652e5f783f4454caabe0f6722802d2570921d414843c854227af9268c4
+oid sha256:5ac169b397e739ab4891bcf75d92fc870af0698d5a104d149ab194bce478f6e0
 size 7416