Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a442be6da9a92f6773b3bd7ac9b003403e2a332bfd3d42ea0dcd0b6d967abbe
+oid sha256:5d9b16ac6175a22bbedc1c33ea81af8500543561a8e9429de2932973eb9081ef
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98238025192746e14a8b6b4b53781b7720c0f4440e7ba0168cb30cf62b192bc3
+oid sha256:1024ffd92d874124cada88654c427e18ea19e270e01fc3ab5cd9ef9e4fdfb4ba
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a7c3c68b087d5376e4c5cf860a8f3c12511fb405ec2d8059ee03119a108d728
+oid sha256:0888b52e2a4249e80620b04c8c28b8b7af505e1d0c8fe1f676f3669eea112abd
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0a6e82c8608f82d881b123ea159fe95bf05ba4ec67b370c4131a2ddb8581d54
+oid sha256:274e8bb901a6c141401f64b7cb6e710f494a233d1e70715f6675d29c5a0fe17f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.566593018117543,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.43305346884666374,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=19-learning_rate=4.1e-05-warmup_ratio=0.100-num_cycles=10/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.5011103749275208,
-      "learning_rate": 0.00019333014963385616,
-      "loss": 0.9833,
+      "grad_norm": 0.8329582214355469,
+      "learning_rate": 2.6272899690440636e-05,
+      "loss": 1.4846,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.30493640899658203,
-      "learning_rate": 0.00016595114383716005,
-      "loss": 0.1736,
+      "grad_norm": 0.6239127516746521,
+      "learning_rate": 8.2058780263589e-07,
+      "loss": 0.2772,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.20391501486301422,
-      "learning_rate": 0.00012396207622088705,
-      "loss": 0.065,
+      "grad_norm": 0.5382785201072693,
+      "learning_rate": 2.5656115903974315e-05,
+      "loss": 0.1826,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.12541060149669647,
-      "learning_rate": 7.66983385109593e-05,
-      "loss": 0.0227,
+      "grad_norm": 0.2955359220504761,
+      "learning_rate": 6.49292407753829e-07,
+      "loss": 0.0981,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.10875938832759857,
-      "learning_rate": 3.466803510220776e-05,
-      "loss": 0.0121,
+      "grad_norm": 0.30408775806427,
+      "learning_rate": 2.503389658442746e-05,
+      "loss": 0.076,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.10840030014514923,
-      "learning_rate": 7.215725637295107e-06,
-      "loss": 0.0084,
+      "grad_norm": 0.2571639120578766,
+      "learning_rate": 4.977412313142637e-07,
+      "loss": 0.0476,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.28355872631073,
-      "learning_rate": 0.0002000300854243022,
-      "loss": 0.0154,
+      "grad_norm": 0.22943373024463654,
+      "learning_rate": 2.4406868253911588e-05,
+      "loss": 0.041,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.1145474836230278,
-      "learning_rate": 0.00018461415129199327,
-      "loss": 0.0275,
+      "grad_norm": 0.23392128944396973,
+      "learning_rate": 3.6608687250542647e-07,
+      "loss": 0.0277,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.1480787992477417,
-      "learning_rate": 0.00015043882563917473,
-      "loss": 0.0093,
+      "grad_norm": 0.22501622140407562,
+      "learning_rate": 2.3775662276804576e-05,
+      "loss": 0.0263,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.08729447424411774,
-      "learning_rate": 0.00010510227795238147,
-      "loss": 0.0059,
+      "grad_norm": 0.1803915649652481,
+      "learning_rate": 2.5446189610677646e-07,
+      "loss": 0.018,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.5108263367211666,
-      "eval_e3_equal": 0.7976137870083959,
+      "eval_all_ligands_equal": 0.43305346884666374,
+      "eval_e3_equal": 0.7778170570039771,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9916924436588599,
-      "eval_e3_heavy_atoms_difference": 0.4874944763588157,
-      "eval_e3_heavy_atoms_difference_norm": 0.006469055588544779,
+      "eval_e3_has_attachment_point(s)": 0.993194874060981,
+      "eval_e3_heavy_atoms_difference": 0.3430844012372956,
+      "eval_e3_heavy_atoms_difference_norm": 0.00442941141209005,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9916924436588599,
-      "eval_has_all_attachment_points": 0.9919575784357049,
-      "eval_has_three_substructures": 0.9999116217410517,
-      "eval_heavy_atoms_difference": 2.0157313300927973,
-      "eval_heavy_atoms_difference_norm": 0.02104730460463736,
-      "eval_linker_equal": 0.7404330534688467,
+      "eval_e3_valid": 0.993194874060981,
+      "eval_has_all_attachment_points": 0.9850640742377376,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 7.4964206805125935,
+      "eval_heavy_atoms_difference_norm": 0.10013163805754127,
+      "eval_linker_equal": 0.6146707909854175,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9988510826336722,
-      "eval_linker_heavy_atoms_difference": 0.546442775077331,
-      "eval_linker_heavy_atoms_difference_norm": 0.006276838082081687,
+      "eval_linker_has_attachment_point(s)": 0.9961997348652232,
+      "eval_linker_heavy_atoms_difference": 0.447635881573133,
+      "eval_linker_heavy_atoms_difference_norm": -0.0017176987217232774,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9988510826336722,
-      "eval_loss": 0.32633018493652344,
-      "eval_num_fragments": 2.9999116217410515,
-      "eval_poi_equal": 0.744675209898365,
+      "eval_linker_valid": 0.9961997348652232,
+      "eval_loss": 0.2785375118255615,
+      "eval_num_fragments": 3.0007954043305345,
+      "eval_poi_equal": 0.7222271321254972,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9913389306230668,
-      "eval_poi_heavy_atoms_difference": -0.07856827220503756,
-      "eval_poi_heavy_atoms_difference_norm": -0.022628294753132334,
+      "eval_poi_has_attachment_point(s)": 0.9169244365885992,
+      "eval_poi_heavy_atoms_difference": 2.4507291206363235,
+      "eval_poi_heavy_atoms_difference_norm": 0.07988863070947928,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9913389306230668,
-      "eval_reassembly": 0.5181617322138754,
-      "eval_reassembly_nostereo": 0.5433495360141405,
-      "eval_runtime": 1536.1175,
-      "eval_samples_per_second": 7.366,
+      "eval_poi_valid": 0.9169244365885992,
+      "eval_reassembly": 0.4402121078214759,
+      "eval_reassembly_nostereo": 0.46425099425541316,
+      "eval_runtime": 1542.5148,
+      "eval_samples_per_second": 7.335,
       "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.982235969951392,
+      "eval_valid": 0.9092355280600972,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.06932828575372696,
-      "learning_rate": 5.868414219788761e-05,
-      "loss": 0.0036,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.04975828155875206,
-      "learning_rate": 2.150452084653951e-05,
-      "loss": 0.0025,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.0682067796587944,
-      "learning_rate": 1.8295257517419904e-06,
-      "loss": 0.002,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.14888513088226318,
-      "learning_rate": 0.0001964414619522753,
-      "loss": 0.0194,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.06711729615926743,
-      "learning_rate": 0.00017284855989881497,
-      "loss": 0.0089,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5022536456031816,
-      "eval_e3_equal": 0.7930181175430844,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9893946089262041,
-      "eval_e3_heavy_atoms_difference": 0.3531595227574017,
-      "eval_e3_heavy_atoms_difference_norm": 0.0022256010654386096,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9893946089262041,
-      "eval_has_all_attachment_points": 0.9910737958462218,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 2.62129916040654,
-      "eval_heavy_atoms_difference_norm": 0.031392457865073144,
-      "eval_linker_equal": 0.7316836058329651,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9990278391515687,
-      "eval_linker_heavy_atoms_difference": 0.6730004418912947,
-      "eval_linker_heavy_atoms_difference_norm": 0.018116166192629755,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9990278391515687,
-      "eval_loss": 0.3227217495441437,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.7464427750773309,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9817057003977022,
-      "eval_poi_heavy_atoms_difference": 0.15298276623950507,
-      "eval_poi_heavy_atoms_difference_norm": -0.012298162059253756,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9817057003977022,
-      "eval_reassembly": 0.5095890410958904,
-      "eval_reassembly_nostereo": 0.5325673884224481,
-      "eval_runtime": 1637.1754,
-      "eval_samples_per_second": 6.911,
-      "eval_steps_per_second": 0.108,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9704816615112682,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.07649541646242142,
-      "learning_rate": 0.00013311209970190936,
-      "loss": 0.0047,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.060988157987594604,
-      "learning_rate": 8.606665290121947e-05,
-      "loss": 0.003,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.04421328753232956,
-      "learning_rate": 4.217179146960002e-05,
-      "loss": 0.002,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.03426754102110863,
-      "learning_rate": 1.1186620561851483e-05,
-      "loss": 0.0014,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.04376823827624321,
-      "learning_rate": 4.5422594786884814e-11,
-      "loss": 0.0012,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.566593018117543,
-      "eval_e3_equal": 0.8154661953159523,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9909854175872735,
-      "eval_e3_heavy_atoms_difference": 0.35466195315952276,
-      "eval_e3_heavy_atoms_difference_norm": 0.003377162233345294,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9909854175872735,
-      "eval_has_all_attachment_points": 0.9925762262483429,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 1.608572691117985,
-      "eval_heavy_atoms_difference_norm": 0.017818672254655762,
-      "eval_linker_equal": 0.8087494476358815,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9996464869642068,
-      "eval_linker_heavy_atoms_difference": 0.29288555015466194,
-      "eval_linker_heavy_atoms_difference_norm": -0.0023201213706328535,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9996464869642068,
-      "eval_loss": 0.34583917260169983,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.7791427308882015,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9923994697304463,
-      "eval_poi_heavy_atoms_difference": -0.043658859920459564,
-      "eval_poi_heavy_atoms_difference_norm": -0.014489169707049073,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9923994697304463,
-      "eval_reassembly": 0.5739284136102519,
-      "eval_reassembly_nostereo": 0.6005302695536898,
-      "eval_runtime": 1544.1434,
-      "eval_samples_per_second": 7.328,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9830313742819267,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3",
+  "trial_name": "trial-number=19-learning_rate=4.1e-05-warmup_ratio=0.100-num_cycles=10",
   "trial_params": {
-    "learning_rate": 0.0002004749428706299,
-    "num_cycles": 3,
-    "warmup_ratio": 0.01
+    "learning_rate": 4.051582532010182e-05,
+    "num_cycles": 10,
+    "warmup_ratio": 0.09999999999999999
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d4dc2f1997c6fb2b97d75f7386d588e59cbc1631d18eba4831404dccc7583fc
+oid sha256:30c24aa130bb589f26c14c76ff3e1402a059dbf6cb4c501dd80b4d1553ff8b07
 size 7480