Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6a5a578f0c1654cb1728c47533deb3ca07393d182847dbda9810c87d4eddda0
+oid sha256:12062b000e1cb8b44553fac4d3e1162aea140c509f7c2dbcfea2aca7cce00046
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99e1a3594eb5da26b473ff86dcb3292d6312354cab75b099645fa4dc3938a31e
+oid sha256:9a20d79c3209721c19c5ab15349c56358582b4c970a52de9820dfcd02eb0bf0e
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78b719d256ecfb1a83a26593745fb074c4c974f8a70967bada23bf0b33d7fd2d
+oid sha256:8bbbef9c5a6186d36adcd5731e7b55b4690c7b3d086f02ba830c3a77bd5cd486
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24e04b6917ab55f6326e4cf96da9b4fd749ccb1f6bee2620e7d3dd1a4888fb83
+oid sha256:655073625197a7e4274d7f9548f06c1fcefb405e98bb15971106f0841041af52
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5699513919575785,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.11842686699072029,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.5925800800323486,
-      "learning_rate": 0.00016371366993372984,
-      "loss": 1.4564,
+      "grad_norm": 1.193555474281311,
+      "learning_rate": 1.9049167867746203e-05,
+      "loss": 2.888,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.2535039484500885,
-      "learning_rate": 9.64787889193764e-05,
-      "loss": 0.08,
+      "grad_norm": 0.8318235278129578,
+      "learning_rate": 2.246394835598448e-05,
+      "loss": 0.5175,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.18813341856002808,
-      "learning_rate": 0.00010153394284957266,
-      "loss": 0.0206,
+      "grad_norm": 0.5888204574584961,
+      "learning_rate": 5.867106147706695e-06,
+      "loss": 0.2372,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.15169543027877808,
-      "learning_rate": 0.00015725808546261587,
-      "loss": 0.0263,
+      "grad_norm": 0.4154653251171112,
+      "learning_rate": 9.20011203733316e-07,
+      "loss": 0.1886,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.10389164090156555,
-      "learning_rate": 4.4808780348991996e-05,
-      "loss": 0.0091,
+      "grad_norm": 0.6848288178443909,
+      "learning_rate": 1.536036183614666e-05,
+      "loss": 0.177,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.17846329510211945,
-      "learning_rate": 0.00020580215204037155,
-      "loss": 0.0157,
+      "grad_norm": 0.5205318927764893,
+      "learning_rate": 2.660216000372886e-05,
+      "loss": 0.1329,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.08230599761009216,
-      "learning_rate": 7.977567091364175e-06,
-      "loss": 0.0088,
+      "grad_norm": 0.2795868217945099,
+      "learning_rate": 1.7062229379525045e-05,
+      "loss": 0.085,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.1894439160823822,
-      "learning_rate": 0.00022824090725901466,
-      "loss": 0.0083,
+      "grad_norm": 0.3321867883205414,
+      "learning_rate": 1.6618734606935415e-06,
+      "loss": 0.0616,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.061632536351680756,
-      "learning_rate": 1.5637709631532583e-06,
-      "loss": 0.0094,
+      "grad_norm": 0.3245985507965088,
+      "learning_rate": 4.48862429344581e-06,
+      "loss": 0.056,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.1319124698638916,
-      "learning_rate": 0.00021816311730162138,
-      "loss": 0.0049,
+      "grad_norm": 0.37055209279060364,
+      "learning_rate": 2.1121191121292532e-05,
+      "loss": 0.0536,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.41140079540433055,
-      "eval_e3_equal": 0.7638532920901459,
+      "eval_all_ligands_equal": 0.11842686699072029,
+      "eval_e3_equal": 0.6632788334069819,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9593460008837826,
-      "eval_e3_heavy_atoms_difference": 1.5961997348652233,
-      "eval_e3_heavy_atoms_difference_norm": 0.039123254458077565,
+      "eval_e3_has_attachment_point(s)": 0.9943437914273089,
+      "eval_e3_heavy_atoms_difference": -0.12567388422448078,
+      "eval_e3_heavy_atoms_difference_norm": -0.018177922333118175,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9593460008837826,
-      "eval_has_all_attachment_points": 0.9722492266902342,
-      "eval_has_three_substructures": 0.9957578435704817,
-      "eval_heavy_atoms_difference": 16.26027397260274,
-      "eval_heavy_atoms_difference_norm": 0.20647141395181035,
-      "eval_linker_equal": 0.6685815289438798,
+      "eval_e3_valid": 0.9943437914273089,
+      "eval_has_all_attachment_points": 0.9742819266460451,
+      "eval_has_three_substructures": 0.9975254087494476,
+      "eval_heavy_atoms_difference": 13.662571807335395,
+      "eval_heavy_atoms_difference_norm": 0.17663075672625272,
+      "eval_linker_equal": 0.22253645603181618,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9886875828546178,
-      "eval_linker_heavy_atoms_difference": 1.2433937251436147,
-      "eval_linker_heavy_atoms_difference_norm": 0.04663004433191857,
+      "eval_linker_has_attachment_point(s)": 0.9926646045072912,
+      "eval_linker_heavy_atoms_difference": 3.407954043305347,
+      "eval_linker_heavy_atoms_difference_norm": 0.168941017421816,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9886875828546178,
-      "eval_loss": 0.29912540316581726,
-      "eval_num_fragments": 2.9957578435704817,
-      "eval_poi_equal": 0.6661953159522758,
+      "eval_linker_valid": 0.9926646045072912,
+      "eval_loss": 0.30737200379371643,
+      "eval_num_fragments": 2.9989394608926205,
+      "eval_poi_equal": 0.560053026955369,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.852938577110031,
-      "eval_poi_heavy_atoms_difference": 5.4296067167476805,
-      "eval_poi_heavy_atoms_difference_norm": 0.16879143962252643,
+      "eval_poi_has_attachment_point(s)": 0.875740167918692,
+      "eval_poi_heavy_atoms_difference": 4.594520547945206,
+      "eval_poi_heavy_atoms_difference_norm": 0.14379318801749935,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.852938577110031,
-      "eval_reassembly": 0.4201502430402121,
-      "eval_reassembly_nostereo": 0.4426866990720283,
-      "eval_runtime": 1668.2452,
-      "eval_samples_per_second": 6.783,
-      "eval_steps_per_second": 0.106,
+      "eval_poi_valid": 0.875740167918692,
+      "eval_reassembly": 0.12152010605391074,
+      "eval_reassembly_nostereo": 0.1375165709235528,
+      "eval_runtime": 1422.6035,
+      "eval_samples_per_second": 7.954,
+      "eval_steps_per_second": 0.124,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8162615996464869,
+      "eval_valid": 0.8695536897923111,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.0758630558848381,
-      "learning_rate": 2.739995113403538e-05,
-      "loss": 0.0125,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.10389214009046555,
-      "learning_rate": 0.00017844822299033737,
-      "loss": 0.0032,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.07643450051546097,
-      "learning_rate": 7.810415658232181e-05,
-      "loss": 0.0109,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.07314460724592209,
-      "learning_rate": 0.00012044362176156312,
-      "loss": 0.0027,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.07255467027425766,
-      "learning_rate": 0.00013918910789064985,
-      "loss": 0.0073,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5348652231551039,
-      "eval_e3_equal": 0.8077772867874503,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9785240830755634,
-      "eval_e3_heavy_atoms_difference": 0.9194874060980999,
-      "eval_e3_heavy_atoms_difference_norm": 0.02738403050962625,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9785240830755634,
-      "eval_has_all_attachment_points": 0.9831197525408749,
-      "eval_has_three_substructures": 0.9993813521873619,
-      "eval_heavy_atoms_difference": 9.151480335837384,
-      "eval_heavy_atoms_difference_norm": 0.12145187956226944,
-      "eval_linker_equal": 0.7649138311975254,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9933716305788776,
-      "eval_linker_heavy_atoms_difference": 0.40450729120636325,
-      "eval_linker_heavy_atoms_difference_norm": 0.003936525020270963,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9933716305788776,
-      "eval_loss": 0.2837851345539093,
-      "eval_num_fragments": 2.999381352187362,
-      "eval_poi_equal": 0.750861688024746,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9107379584622183,
-      "eval_poi_heavy_atoms_difference": 2.652938577110031,
-      "eval_poi_heavy_atoms_difference_norm": 0.08625993996859925,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9107379584622183,
-      "eval_reassembly": 0.5424657534246575,
-      "eval_reassembly_nostereo": 0.562704374723818,
-      "eval_runtime": 1534.4399,
-      "eval_samples_per_second": 7.374,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.887582854617764,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.06594572961330414,
-      "learning_rate": 6.0722472746074165e-05,
-      "loss": 0.0025,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.1494133025407791,
-      "learning_rate": 0.0001932015238178336,
-      "loss": 0.0095,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.040824271738529205,
-      "learning_rate": 1.634838960794289e-05,
-      "loss": 0.0045,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.21492616832256317,
-      "learning_rate": 0.00022470889823098504,
-      "loss": 0.0041,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.057117585092782974,
-      "learning_rate": 0.0,
-      "loss": 0.0062,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5699513919575785,
-      "eval_e3_equal": 0.8192664604507292,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9812638091029606,
-      "eval_e3_heavy_atoms_difference": 0.6555899248784799,
-      "eval_e3_heavy_atoms_difference_norm": 0.01675108799959528,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9812638091029606,
-      "eval_has_all_attachment_points": 0.989041095890411,
-      "eval_has_three_substructures": 0.9998232434821034,
-      "eval_heavy_atoms_difference": 7.600265134776845,
-      "eval_heavy_atoms_difference_norm": 0.10056611800605245,
-      "eval_linker_equal": 0.8079540433053469,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9977905435262925,
-      "eval_linker_heavy_atoms_difference": 0.27220503756076003,
-      "eval_linker_heavy_atoms_difference_norm": -0.002432511049529868,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9977905435262925,
-      "eval_loss": 0.26366686820983887,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.7761378700839593,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9255855059655325,
-      "eval_poi_heavy_atoms_difference": 2.2732655766681398,
-      "eval_poi_heavy_atoms_difference_norm": 0.07230797500159608,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9255855059655325,
-      "eval_reassembly": 0.57870083959346,
-      "eval_reassembly_nostereo": 0.6014140521431728,
-      "eval_runtime": 1536.0555,
-      "eval_samples_per_second": 7.366,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9069376933274415,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500",
+  "trial_name": "trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4.000",
   "trial_params": {
-    "learning_rate": 0.00022887171056735432,
-    "num_cycles": 8.5,
-    "warmup_ratio": 0.06999999999999999
+    "learning_rate": 2.663073667910919e-05,
+    "num_cycles": 4.0,
+    "warmup_ratio": 0.02
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7ab3268adb85b02359ec17e701405643275b8826a21c0e929b050857cbff37e
+oid sha256:31978025e93efda8b0ad609eee57a99711e64540e52af23213d222e4dc7d2484
 size 7416