Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bb81a97041242cc2fc31203b495a4bebce526ab5850a33935de548ecba8f405
+oid sha256:48b0e0e392de30e27c1678fa90a3714f2ed69176dde100c22e0602c0120aef59
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db1fa98ce0cd0b5bab5ec7deb8147d179b46673a3ebff6e8680af47246ee7161
+oid sha256:518e736af647fcc70835e741267a15d776ec7962141ca9b539a19357ff03006f
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb8e3701c78801c3682fb8fe005ef92c2dac8de0928d1ca26b5d0b626cec9b96
+oid sha256:7512f3014b8688aecadf72081c095e15a11bfd4f0c6939adc2147da05c208a7f
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d222911452edcfb89948637f60e982596ac5a98020aa9b5c2785b3504ae6ab43
+oid sha256:8a4a11b800eb83e700cf44a26c7346fc77b30b99dcb2df7384323250a5ed6a5e
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=9/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.2691117984975696,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.4272928535938263,
-      "learning_rate": 0.0001622713265528645,
-      "loss": 0.9552,
+      "grad_norm": 0.9225704073905945,
+      "learning_rate": 2.4950994668529118e-05,
+      "loss": 1.6862,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.4963849186897278,
-      "learning_rate": 1.834337670199748e-05,
-      "loss": 0.1981,
+      "grad_norm": 0.6497099995613098,
+      "learning_rate": 1.8846772251455575e-05,
+      "loss": 0.3201,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.2832266688346863,
-      "learning_rate": 0.00018962209765012623,
-      "loss": 0.1843,
+      "grad_norm": 0.5509499907493591,
+      "learning_rate": 1.058829211586969e-05,
+      "loss": 0.1784,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.3434854745864868,
-      "learning_rate": 3.974361254751191e-05,
-      "loss": 0.167,
+      "grad_norm": 0.3397536873817444,
+      "learning_rate": 3.391897681746473e-06,
+      "loss": 0.135,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.30111342668533325,
-      "learning_rate": 0.0002108817265406588,
-      "loss": 0.16,
+      "grad_norm": 0.4305669665336609,
+      "learning_rate": 6.029301880940131e-08,
+      "loss": 0.1211,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.33496615290641785,
-      "learning_rate": 6.719496968763881e-05,
-      "loss": 0.1641,
+      "grad_norm": 0.35494357347488403,
+      "learning_rate": 2.4739733516006596e-05,
+      "loss": 0.1049,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.256843626499176,
-      "learning_rate": 0.0002243278845276328,
-      "loss": 0.1517,
+      "grad_norm": 0.2973407208919525,
+      "learning_rate": 1.8459695994350122e-05,
+      "loss": 0.0658,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.27516108751296997,
-      "learning_rate": 9.847350242639213e-05,
-      "loss": 0.1643,
+      "grad_norm": 0.3330534100532532,
+      "learning_rate": 1.0176151273905278e-05,
+      "loss": 0.0475,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.45082804560661316,
-      "learning_rate": 4.667090302646467e-10,
-      "loss": 0.1534,
+      "grad_norm": 0.2717476785182953,
+      "learning_rate": 3.115204414956426e-06,
+      "loss": 0.0391,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.2763317823410034,
-      "learning_rate": 0.00013104521006187479,
-      "loss": 0.1611,
+      "grad_norm": 0.26711612939834595,
+      "learning_rate": 2.680814436355032e-08,
+      "loss": 0.0362,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.014140521431727796,
+      "eval_all_ligands_equal": 0.2691117984975696,
+      "eval_e3_equal": 0.7353071144498453,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 8.767211665930182,
-      "eval_e3_heavy_atoms_difference_norm": 0.24450330585389068,
+      "eval_e3_has_attachment_point(s)": 0.9956694653115333,
+      "eval_e3_heavy_atoms_difference": -0.16137870083959346,
+      "eval_e3_heavy_atoms_difference_norm": -0.018238988976364858,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 11.208661069376934,
-      "eval_heavy_atoms_difference_norm": 0.11492366081892867,
-      "eval_linker_equal": 0.0014140521431727795,
+      "eval_e3_valid": 0.9956694653115333,
+      "eval_has_all_attachment_points": 0.9850640742377376,
+      "eval_has_three_substructures": 0.999116217410517,
+      "eval_heavy_atoms_difference": 7.798585947856827,
+      "eval_heavy_atoms_difference_norm": 0.10333166546592974,
+      "eval_linker_equal": 0.39717189571365447,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -1.8817498895271763,
-      "eval_linker_heavy_atoms_difference_norm": -0.3738792733413414,
+      "eval_linker_has_attachment_point(s)": 0.9960229783473266,
+      "eval_linker_heavy_atoms_difference": 0.8761820592134335,
+      "eval_linker_heavy_atoms_difference_norm": 0.013908912707481406,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.49358931183815,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.002209456473707468,
+      "eval_linker_valid": 0.9960229783473266,
+      "eval_loss": 0.28606870770454407,
+      "eval_num_fragments": 2.9998232434821035,
+      "eval_poi_equal": 0.671321254971277,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": 4.3231992929739285,
-      "eval_poi_heavy_atoms_difference_norm": 0.015593075838489463,
+      "eval_poi_has_attachment_point(s)": 0.9096774193548387,
+      "eval_poi_heavy_atoms_difference": 2.651524524966858,
+      "eval_poi_heavy_atoms_difference_norm": 0.0825384289930311,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1140.3291,
-      "eval_samples_per_second": 9.923,
-      "eval_steps_per_second": 0.155,
+      "eval_poi_valid": 0.9096774193548387,
+      "eval_reassembly": 0.2733539549270879,
+      "eval_reassembly_nostereo": 0.2933274414494034,
+      "eval_runtime": 1568.5175,
+      "eval_samples_per_second": 7.214,
+      "eval_steps_per_second": 0.113,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
+      "eval_valid": 0.9043747238179408,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.40334638953208923,
-      "learning_rate": 4.72798198719592e-06,
-      "loss": 0.1524,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.25897476077079773,
-      "learning_rate": 0.00016227132655286433,
-      "loss": 0.1571,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.42913806438446045,
-      "learning_rate": 1.8343376701997453e-05,
-      "loss": 0.1525,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.30887413024902344,
-      "learning_rate": 0.0001896220976501263,
-      "loss": 0.1542,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.3643406927585602,
-      "learning_rate": 3.974361254751187e-05,
-      "loss": 0.1528,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.004684047724259832,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -4.8929739284136105,
-      "eval_e3_heavy_atoms_difference_norm": -0.28970659073238636,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -19.629253203711887,
-      "eval_heavy_atoms_difference_norm": -0.3094509206668312,
-      "eval_linker_equal": 0.002209456473707468,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -3.4003535130357934,
-      "eval_linker_heavy_atoms_difference_norm": -0.48975559440086974,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.49861598014831543,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0017675651789659744,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -11.335925762262484,
-      "eval_poi_heavy_atoms_difference_norm": -0.5637834471539719,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1775.4646,
-      "eval_samples_per_second": 6.373,
-      "eval_steps_per_second": 0.1,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.23480825126171112,
-      "learning_rate": 0.00021088172654065888,
-      "loss": 0.1496,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.397032767534256,
-      "learning_rate": 6.719496968763895e-05,
-      "loss": 0.1523,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.23358654975891113,
-      "learning_rate": 0.0002243278845276328,
-      "loss": 0.1473,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.35571378469467163,
-      "learning_rate": 9.847350242639196e-05,
-      "loss": 0.1557,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.4071267247200012,
-      "learning_rate": 1.8668323142670654e-09,
-      "loss": 0.1463,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.006716747680070702,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -5.479010163499779,
-      "eval_e3_heavy_atoms_difference_norm": -0.31248932677521146,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -16.36579761378701,
-      "eval_heavy_atoms_difference_norm": -0.2639458185695727,
-      "eval_linker_equal": 0.0013256738842244808,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -2.0755634114007955,
-      "eval_linker_heavy_atoms_difference_norm": -0.38850172612130895,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5050681829452515,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0013256738842244808,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -8.811224038886435,
-      "eval_poi_heavy_atoms_difference_norm": -0.46859326736968265,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1641.1843,
-      "eval_samples_per_second": 6.894,
-      "eval_steps_per_second": 0.108,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=9",
+  "trial_name": "trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4",
   "trial_params": {
-    "learning_rate": 0.00022887171056735432,
-    "num_cycles": 9,
-    "warmup_ratio": 0.06999999999999999
+    "learning_rate": 2.663073667910919e-05,
+    "num_cycles": 4,
+    "warmup_ratio": 0.02
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6719a23f4a4763d457ab5fb9dc58cde2b4a0dce1744ce9a3e4a62b3b819baad1
+oid sha256:142086f5e312a0f106951e266b96c414d0c14b22cdec248257e38aa9958859c4
 size 7480