Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2655a7c5a6f3b44280a8b2d96529887f85baa3433f1588eb504291c7bb228836
+oid sha256:52c68f67a156c4b6328a1db733c8646323562cac197d158692ed5af485b1be65
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb06b002543c91ea316a35c24a2821d6397a7c06b11933b2c6ccf7ddbb23effc
+oid sha256:dc0be0c251415bfc2c44d5d6913c5160be7d34569fc6f937ce67940be0394c0d
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edfb0b0889cbd2250b35734bcab9cc469c772472998ce7fd321d378f0e09ea2f
+oid sha256:4863aade451900f1bfe0ce71323883fceafafc0b86d1937632b5d4e867a7d5a7
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6092aaf0f5bb75d5e667503b11558e227dd128e7e1ea91b5de70f7cbae1e6e1c
+oid sha256:4ba0356b80a479f17dfa8c1fc56665b7f10a045f404714fdb1907f8d791a3bf9
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.1942554131683606,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5010623229461756,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
-  "is_hyper_param_search": true,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.0838514566421509,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 1.9521,
+      "grad_norm": 0.7027302384376526,
+      "learning_rate": 5e-05,
+      "loss": 0.9638,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.154284954071045,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.727,
+      "grad_norm": 0.5134932994842529,
+      "learning_rate": 5e-05,
+      "loss": 0.1891,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.0525119304656982,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.4868,
+      "grad_norm": 0.27161791920661926,
+      "learning_rate": 5e-05,
+      "loss": 0.0815,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.8092649579048157,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.3584,
+      "grad_norm": 0.27095702290534973,
+      "learning_rate": 5e-05,
+      "loss": 0.0463,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.8515424728393555,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.2778,
+      "grad_norm": 0.22146300971508026,
+      "learning_rate": 5e-05,
+      "loss": 0.0303,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.5962498784065247,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.2258,
+      "grad_norm": 0.20155303180217743,
+      "learning_rate": 5e-05,
+      "loss": 0.0225,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.44913268089294434,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.1889,
+      "grad_norm": 0.15531788766384125,
+      "learning_rate": 5e-05,
+      "loss": 0.0172,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.5444086194038391,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.1641,
+      "grad_norm": 0.18832802772521973,
+      "learning_rate": 5e-05,
+      "loss": 0.0142,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.48740506172180176,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.1436,
+      "grad_norm": 0.141372412443161,
+      "learning_rate": 5e-05,
+      "loss": 0.0117,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.4699327051639557,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.1276,
+      "grad_norm": 0.1557079702615738,
+      "learning_rate": 5e-05,
+      "loss": 0.0107,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.00821917808219178,
-      "eval_e3_equal": 0.426336721166593,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9975254087494476,
-      "eval_e3_heavy_atoms_difference": -0.7306230667255855,
-      "eval_e3_heavy_atoms_difference_norm": -0.043387643547083826,
+      "eval_all_ligands_equal": 0.5010623229461756,
+      "eval_e3_equal": 0.7893944759206799,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9859242209631728,
+      "eval_e3_heavy_atoms_difference": 0.44741501416430596,
+      "eval_e3_heavy_atoms_difference_norm": 0.010527970436043508,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9975254087494476,
-      "eval_has_all_attachment_points": 0.9786124613345117,
-      "eval_has_three_substructures": 0.9981440565620857,
-      "eval_heavy_atoms_difference": 10.570216526734423,
-      "eval_heavy_atoms_difference_norm": 0.13624537211132287,
-      "eval_linker_equal": 0.028369421122403888,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9939019001325674,
-      "eval_linker_heavy_atoms_difference": 3.5113566062748562,
-      "eval_linker_heavy_atoms_difference_norm": 0.143718666575472,
+      "eval_e3_valid": 0.9859242209631728,
+      "eval_has_all_attachment_points": 0.9880488668555241,
+      "eval_has_three_substructures": 0.9992917847025495,
+      "eval_heavy_atoms_difference": 4.677850566572238,
+      "eval_heavy_atoms_difference_norm": 0.06169203765049902,
+      "eval_linker_equal": 0.7242386685552408,
+      "eval_linker_graph_edit_distance": Infinity,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9977868271954674,
+      "eval_linker_heavy_atoms_difference": 0.5833923512747875,
+      "eval_linker_heavy_atoms_difference_norm": 0.017295105295040032,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9939019001325674,
-      "eval_loss": 0.33331799507141113,
-      "eval_num_fragments": 2.9986743261157756,
-      "eval_poi_equal": 0.45223155103844453,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9035793194874061,
-      "eval_poi_heavy_atoms_difference": 3.2608042421564294,
-      "eval_poi_heavy_atoms_difference_norm": 0.0900747544488257,
+      "eval_linker_valid": 0.9977868271954674,
+      "eval_loss": 0.30077189207077026,
+      "eval_num_fragments": 3.0001770538243626,
+      "eval_poi_equal": 0.7466359773371105,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9623760623229461,
+      "eval_poi_heavy_atoms_difference": 1.126150849858357,
+      "eval_poi_heavy_atoms_difference_norm": 0.03167513129582722,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9035793194874061,
-      "eval_reassembly": 0.008484312859036678,
-      "eval_reassembly_nostereo": 0.011489173663278833,
-      "eval_runtime": 1538.0562,
-      "eval_samples_per_second": 7.357,
-      "eval_steps_per_second": 0.115,
+      "eval_poi_valid": 0.9623760623229461,
+      "eval_reassembly": 0.5097379603399433,
+      "eval_reassembly_nostereo": 0.5343484419263456,
+      "eval_runtime": 2394.3813,
+      "eval_samples_per_second": 4.718,
+      "eval_steps_per_second": 0.074,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8996022978347327,
+      "eval_valid": 0.9481232294617564,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.4523199796676636,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.1138,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.4032362997531891,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.1022,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.39051586389541626,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0913,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.4568197429180145,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0824,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.4104657769203186,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0734,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.09226690234202387,
-      "eval_e3_equal": 0.5859478568272205,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9982324348210341,
-      "eval_e3_heavy_atoms_difference": -0.40927971718957135,
-      "eval_e3_heavy_atoms_difference_norm": -0.028732096583262583,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9982324348210341,
-      "eval_has_all_attachment_points": 0.9905435262925321,
-      "eval_has_three_substructures": 0.9989394608926204,
-      "eval_heavy_atoms_difference": 4.900309323906319,
-      "eval_heavy_atoms_difference_norm": 0.06346159241850167,
-      "eval_linker_equal": 0.17065841802916482,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9969951391957579,
-      "eval_linker_heavy_atoms_difference": 0.9197525408749447,
-      "eval_linker_heavy_atoms_difference_norm": -0.004279007320364481,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9969951391957579,
-      "eval_loss": 0.30911630392074585,
-      "eval_num_fragments": 2.999646486964207,
-      "eval_poi_equal": 0.5859478568272205,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9417587273530711,
-      "eval_poi_heavy_atoms_difference": 1.571895713654441,
-      "eval_poi_heavy_atoms_difference_norm": 0.04295033310659916,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9417587273530711,
-      "eval_reassembly": 0.09368095448519664,
-      "eval_reassembly_nostereo": 0.11162174105170128,
-      "eval_runtime": 1561.5722,
-      "eval_samples_per_second": 7.246,
-      "eval_steps_per_second": 0.113,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9394608926204154,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.3263263702392578,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0658,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.31009504199028015,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0597,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.3627758324146271,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0542,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.2838134765625,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0493,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.3240777254104614,
-      "learning_rate": 7.729384753693693e-06,
-      "loss": 0.0454,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.1942554131683606,
-      "eval_e3_equal": 0.6628369421122404,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.997348652231551,
-      "eval_e3_heavy_atoms_difference": -0.17587273530711445,
-      "eval_e3_heavy_atoms_difference_norm": -0.018808788002665174,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.997348652231551,
-      "eval_has_all_attachment_points": 0.9904551480335837,
-      "eval_has_three_substructures": 0.9993813521873619,
-      "eval_heavy_atoms_difference": 5.831462660185594,
-      "eval_heavy_atoms_difference_norm": 0.07443880795953683,
-      "eval_linker_equal": 0.3456473707467963,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9966416261599647,
-      "eval_linker_heavy_atoms_difference": 0.8881131241714538,
-      "eval_linker_heavy_atoms_difference_norm": 0.011275755194650126,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9966416261599647,
-      "eval_loss": 0.3081931471824646,
-      "eval_num_fragments": 2.9999116217410515,
-      "eval_poi_equal": 0.6042421564295183,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.942907644719399,
-      "eval_poi_heavy_atoms_difference": 2.1913389306230666,
-      "eval_poi_heavy_atoms_difference_norm": 0.06304879570876758,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.942907644719399,
-      "eval_reassembly": 0.19787892178524083,
-      "eval_reassembly_nostereo": 0.23057887759611137,
-      "eval_runtime": 1505.1145,
-      "eval_samples_per_second": 7.518,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9386654882898807,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570",
-  "trial_params": {
-    "factor": 0.5700000000000001,
-    "learning_rate": 7.729384753693693e-06,
-    "min_lr": 3.4099230386483844e-10,
-    "warmup_ratio": 0.04
-  }
+  "trial_name": null,
+  "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3834bc12963422092c70312d13bbac57fcc1945c5f1a3d131e77cedfa4502dec
-size 7544
+oid sha256:ad33a0498995af6c669f7364548479657f7d6eaa0e9abe34126ea1bef6adebd5
+size 7480