Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:519b0b5e6ed5910caa7fcf55b70716e5b7284b980bcb820044892272b34d4876
+oid sha256:50011882f6e94485ecd911a464934f117f78064da0d44af2faa0a580d2dc69bc
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff00ea088844a1946680a75e7a843888c2a49cf8d11b53f3bbb34172261f14fa
+oid sha256:d19696c9fc94f66c580b32203b546d48c2941b3566f35054fb4baaa703129588
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69f7d504a21439ea8908c655a4fc0232d07a90a6362f2937bed94c74964be650
+oid sha256:1d22c10d7550e065c81ef680acc47320cb8967eab76c92c9bc164c3ac3c2f176
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cecf87f53d34ddc9473e4a30679ce4f910e5a6dd762c6655ed8a15f6bca7ece4
+oid sha256:6828ba4e0a02ba8ef04fc15c3250ecdbde02ded1248c840d960be581184449ca
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=5-learning_rate=1.3e-06-warmup_ratio=0.080-num_cycles=4.000/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5443216968625718,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=6-learning_rate=1.6e-04-warmup_ratio=0.010-num_cycles=9.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 4.2125678062438965,
-      "learning_rate": 9.653298476903908e-07,
-      "loss": 6.6359,
+      "grad_norm": 0.7057588696479797,
+      "learning_rate": 0.00011380321317599481,
+      "loss": 1.6278,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 2.408616542816162,
-      "learning_rate": 1.1383762270122183e-06,
-      "loss": 2.6568,
+      "grad_norm": 0.30460575222969055,
+      "learning_rate": 5.145809841154338e-05,
+      "loss": 0.1097,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.875552773475647,
-      "learning_rate": 2.973196899345272e-07,
-      "loss": 1.588,
+      "grad_norm": 0.27816638350486755,
+      "learning_rate": 0.00011257406636063424,
+      "loss": 0.0351,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 2.1072449684143066,
-      "learning_rate": 4.662220845232186e-08,
-      "loss": 1.4699,
+      "grad_norm": 0.1406199336051941,
+      "learning_rate": 4.173670662244571e-05,
+      "loss": 0.0288,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.682381510734558,
-      "learning_rate": 7.783970331251584e-07,
-      "loss": 1.438,
+      "grad_norm": 0.20178978145122528,
+      "learning_rate": 0.00012197553515461338,
+      "loss": 0.012,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.580583095550537,
-      "learning_rate": 1.3480829841452437e-06,
-      "loss": 1.294,
+      "grad_norm": 0.12134135514497757,
+      "learning_rate": 3.269762070727056e-05,
+      "loss": 0.0164,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.402235746383667,
-      "learning_rate": 8.646403560799816e-07,
-      "loss": 1.1103,
+      "grad_norm": 0.18423935770988464,
+      "learning_rate": 0.00013061141493825746,
+      "loss": 0.0068,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.4020792245864868,
-      "learning_rate": 8.421659496257073e-08,
-      "loss": 1.0136,
+      "grad_norm": 0.06757809221744537,
+      "learning_rate": 2.4503949306739228e-05,
+      "loss": 0.0134,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.4105451107025146,
-      "learning_rate": 2.2746416198410507e-07,
-      "loss": 0.9933,
+      "grad_norm": 0.11270517110824585,
+      "learning_rate": 0.00013832587285123472,
+      "loss": 0.005,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 1.4410276412963867,
-      "learning_rate": 1.0703310690373619e-06,
-      "loss": 0.9689,
+      "grad_norm": 0.08828501403331757,
+      "learning_rate": 1.7303545712345613e-05,
+      "loss": 0.0099,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.02421564295183385,
+      "eval_all_ligands_equal": 0.5443216968625718,
+      "eval_e3_equal": 0.8076005302695537,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.7856827220503756,
-      "eval_e3_heavy_atoms_difference": 7.01767565178966,
-      "eval_e3_heavy_atoms_difference_norm": 0.193812682287618,
+      "eval_e3_has_attachment_point(s)": 0.9897481219619974,
+      "eval_e3_heavy_atoms_difference": 0.45161290322580644,
+      "eval_e3_heavy_atoms_difference_norm": 0.00716518262897802,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.7856827220503756,
-      "eval_has_all_attachment_points": 0.47989394608926206,
-      "eval_has_three_substructures": 0.7911621741051701,
-      "eval_heavy_atoms_difference": 71.59911621741051,
-      "eval_heavy_atoms_difference_norm": 0.9386096937863235,
-      "eval_linker_equal": 0.0007954043305346885,
+      "eval_e3_valid": 0.9897481219619974,
+      "eval_has_all_attachment_points": 0.9868316394167035,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 6.609102960671675,
+      "eval_heavy_atoms_difference_norm": 0.08723273195861195,
+      "eval_linker_equal": 0.776933274414494,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.6998674326115776,
-      "eval_linker_heavy_atoms_difference": 11.983208130799824,
-      "eval_linker_heavy_atoms_difference_norm": 0.6700703059837816,
+      "eval_linker_has_attachment_point(s)": 0.9962881131241714,
+      "eval_linker_heavy_atoms_difference": 0.4151126822801591,
+      "eval_linker_heavy_atoms_difference_norm": 0.00679634617674321,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.6998674326115776,
-      "eval_loss": 0.9760982990264893,
-      "eval_num_fragments": 3.1617322138753865,
-      "eval_poi_equal": 0.0,
+      "eval_linker_valid": 0.9962881131241714,
+      "eval_loss": 0.2718093991279602,
+      "eval_num_fragments": 2.999734865223155,
+      "eval_poi_equal": 0.7680070702607159,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.15687140963323024,
-      "eval_poi_heavy_atoms_difference": 29.274502872293414,
-      "eval_poi_heavy_atoms_difference_norm": 0.9270241865988441,
+      "eval_poi_has_attachment_point(s)": 0.9324790101634998,
+      "eval_poi_heavy_atoms_difference": 2.0205921343349535,
+      "eval_poi_heavy_atoms_difference_norm": 0.06507441168533995,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.15687140963323024,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 2252.5623,
-      "eval_samples_per_second": 5.023,
-      "eval_steps_per_second": 0.079,
+      "eval_poi_valid": 0.9324790101634998,
+      "eval_reassembly": 0.5524524966858153,
+      "eval_reassembly_nostereo": 0.5774635439681838,
+      "eval_runtime": 1524.2847,
+      "eval_samples_per_second": 7.423,
+      "eval_steps_per_second": 0.116,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.1456473707467963,
+      "eval_valid": 0.9205479452054794,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 1.3164554834365845,
-      "learning_rate": 1.2944981979576419e-06,
-      "loss": 0.8841,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 1.277483344078064,
-      "learning_rate": 5.493481389893816e-07,
-      "loss": 0.7983,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 1.2130144834518433,
-      "learning_rate": 3.621329061505025e-10,
-      "loss": 0.7606,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 1.3282479047775269,
-      "learning_rate": 5.062033225695726e-07,
-      "loss": 0.7549,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 1.2423975467681885,
-      "learning_rate": 1.275690911215575e-06,
-      "loss": 0.7257,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.04021210782147592,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.942907644719399,
-      "eval_e3_heavy_atoms_difference": 2.112063632346443,
-      "eval_e3_heavy_atoms_difference_norm": 0.02455922153801519,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.942907644719399,
-      "eval_has_all_attachment_points": 0.6668139637649139,
-      "eval_has_three_substructures": 0.9450287229341582,
-      "eval_heavy_atoms_difference": 65.39646486964207,
-      "eval_heavy_atoms_difference_norm": 0.8497526919879462,
-      "eval_linker_equal": 0.0009721608484312859,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.905346884666372,
-      "eval_linker_heavy_atoms_difference": 9.564030048608043,
-      "eval_linker_heavy_atoms_difference_norm": 0.49115750761012605,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.905346884666372,
-      "eval_loss": 0.7647107839584351,
-      "eval_num_fragments": 3.0371188687582853,
-      "eval_poi_equal": 0.0,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.2851082633672117,
-      "eval_poi_heavy_atoms_difference": 26.304728236853734,
-      "eval_poi_heavy_atoms_difference_norm": 0.8131354921263185,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.2851082633672117,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1436.6134,
-      "eval_samples_per_second": 7.876,
-      "eval_steps_per_second": 0.123,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.2703490941228458,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 1.3089553117752075,
-      "learning_rate": 1.105277588096101e-06,
-      "loss": 0.6684,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 1.2422657012939453,
-      "learning_rate": 2.61505009999899e-07,
-      "loss": 0.6209,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 1.2623106241226196,
-      "learning_rate": 6.41088454873336e-08,
-      "loss": 0.6078,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 1.4066320657730103,
-      "learning_rate": 8.218343250943806e-07,
-      "loss": 0.6022,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 1.036487102508545,
-      "learning_rate": 1.3495311270711663e-06,
-      "loss": 0.5743,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0695536897923111,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9741051701281485,
-      "eval_e3_heavy_atoms_difference": 0.2054794520547945,
-      "eval_e3_heavy_atoms_difference_norm": -0.035548164302134005,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9741051701281485,
-      "eval_has_all_attachment_points": 0.7921343349536014,
-      "eval_has_three_substructures": 0.975430844012373,
-      "eval_heavy_atoms_difference": 60.76076005302696,
-      "eval_heavy_atoms_difference_norm": 0.7850994356131387,
-      "eval_linker_equal": 0.0015024304021210783,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.945293857711003,
-      "eval_linker_heavy_atoms_difference": 8.344321696862572,
-      "eval_linker_heavy_atoms_difference_norm": 0.407239177531272,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.945293857711003,
-      "eval_loss": 0.6423223614692688,
-      "eval_num_fragments": 3.0143172779496243,
-      "eval_poi_equal": 0.0032699955810870525,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.31436146707909857,
-      "eval_poi_heavy_atoms_difference": 25.11913389306231,
-      "eval_poi_heavy_atoms_difference_norm": 0.7623433898182805,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.31436146707909857,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1349.2935,
-      "eval_samples_per_second": 8.386,
-      "eval_steps_per_second": 0.131,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.30419796730004417,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=5-learning_rate=1.3e-06-warmup_ratio=0.080-num_cycles=4.000",
+  "trial_name": "trial-number=6-learning_rate=1.6e-04-warmup_ratio=0.010-num_cycles=9.500",
   "trial_params": {
-    "learning_rate": 1.3495311270711663e-06,
-    "num_cycles": 4.0,
-    "warmup_ratio": 0.08
+    "learning_rate": 0.00015909689202004074,
+    "num_cycles": 9.5,
+    "warmup_ratio": 0.01
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71d2586a39a8b7d80743d047b8033a47d036f0685212eab19f9ffbf7918ea36d
+oid sha256:90b600b0ca134f0406f3fc4859fcf52657b085668ffed4c0a674e3f597fcdfbc
 size 7416