Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d4c9de4aefc20647fa99d39bfde363ec849a5042c894242e07adf141ce4454f
+oid sha256:d86756848fc92b8f32c629b38001ab010f12e277f6b4bb950ef9c3a84f2c58d1
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a81078ed187a174a02d150e5fd269f40e45056ec47b64bb0f4df9a38bbe73817
+oid sha256:23ee4de09e4ef9d0b33063d422cecdfc965568d4d20f79b1b4459cec3f7b13fc
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca8e8a22fe037de43f552be26fde69c957c4294d3f5e193809ee0c8388ae3d1c
+oid sha256:6bbd0f47bca72adc058e11195c706225cdf2c0113557c23ad307c5bc2e0b11ee
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:261c4ef52e2149f96eedc5ad4cd9432f645fe92a07619798bca1a0c85e926c74
+oid sha256:280e56c73026ae31058c531ae776e6904f00046b1ce3d2a28f1f305ef0adbe32
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5527176314626602,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=23-learning_rate=5.8e-05-warmup_ratio=0.050-min_lr=0.000-factor=0.800/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.2837825894829872,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.6019228100776672,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.9116,
+      "grad_norm": 1.0362313985824585,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 1.5088,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.4120740592479706,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.1513,
+      "grad_norm": 0.7315112948417664,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.4704,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.29680919647216797,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0565,
+      "grad_norm": 0.7089930176734924,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.2658,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.22217430174350739,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0309,
+      "grad_norm": 0.517996609210968,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.1763,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.1675247699022293,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0195,
+      "grad_norm": 0.5622127056121826,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.1252,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.19688111543655396,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.014,
+      "grad_norm": 0.438522070646286,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.092,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.17143021523952484,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.011,
+      "grad_norm": 0.3132740259170532,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0692,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.12186994403600693,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0089,
+      "grad_norm": 0.3049832582473755,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0546,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.14182540774345398,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0072,
+      "grad_norm": 0.32669779658317566,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0438,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.16419431567192078,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0068,
+      "grad_norm": 0.34408244490623474,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0364,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.517454706142289,
-      "eval_e3_equal": 0.798232434821034,
+      "eval_all_ligands_equal": 0.2837825894829872,
+      "eval_e3_equal": 0.747326557666814,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9901016349977906,
-      "eval_e3_heavy_atoms_difference": 0.5490057445868316,
-      "eval_e3_heavy_atoms_difference_norm": 0.011477999514524074,
+      "eval_e3_has_attachment_point(s)": 0.9939019001325674,
+      "eval_e3_heavy_atoms_difference": 0.1380468404772426,
+      "eval_e3_heavy_atoms_difference_norm": -0.004540464395693658,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9901016349977906,
-      "eval_has_all_attachment_points": 0.9792311091471498,
-      "eval_has_three_substructures": 0.9985859478568272,
-      "eval_heavy_atoms_difference": 8.138930623066726,
-      "eval_heavy_atoms_difference_norm": 0.10671091458695267,
-      "eval_linker_equal": 0.747326557666814,
+      "eval_e3_valid": 0.9939019001325674,
+      "eval_has_all_attachment_points": 0.9800265134776844,
+      "eval_has_three_substructures": 0.9987627043747238,
+      "eval_heavy_atoms_difference": 9.344498453380469,
+      "eval_heavy_atoms_difference_norm": 0.12324719766631663,
+      "eval_linker_equal": 0.4260715863897481,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.994962439239947,
-      "eval_linker_heavy_atoms_difference": 0.4178524083075563,
-      "eval_linker_heavy_atoms_difference_norm": 0.004257830368338666,
+      "eval_linker_has_attachment_point(s)": 0.993813521873619,
+      "eval_linker_heavy_atoms_difference": 1.339637649138312,
+      "eval_linker_heavy_atoms_difference_norm": 0.04383051599945908,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.994962439239947,
-      "eval_loss": 0.30216675996780396,
-      "eval_num_fragments": 3.000353513035793,
-      "eval_poi_equal": 0.7433495360141406,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9159522757401679,
-      "eval_poi_heavy_atoms_difference": 2.62076889085285,
-      "eval_poi_heavy_atoms_difference_norm": 0.08558461375931094,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9159522757401679,
-      "eval_reassembly": 0.5247901016349978,
-      "eval_reassembly_nostereo": 0.5511268228015908,
-      "eval_runtime": 1580.4016,
-      "eval_samples_per_second": 7.16,
-      "eval_steps_per_second": 0.112,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9047282368537339,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.1013902872800827,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.007,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.095343217253685,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.005,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.0741553008556366,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0046,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.07309788465499878,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0041,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.12277767062187195,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0039,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5486522315510385,
-      "eval_e3_equal": 0.8102518780380027,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9855943437914273,
-      "eval_e3_heavy_atoms_difference": 0.34962439239946974,
-      "eval_e3_heavy_atoms_difference_norm": 0.005698624810067466,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9855943437914273,
-      "eval_has_all_attachment_points": 0.984887317719841,
-      "eval_has_three_substructures": 0.9994697304463102,
-      "eval_heavy_atoms_difference": 7.842775077330977,
-      "eval_heavy_atoms_difference_norm": 0.10476501334640989,
-      "eval_linker_equal": 0.7788775961113567,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9950508174988952,
-      "eval_linker_heavy_atoms_difference": 0.4073353954927088,
-      "eval_linker_heavy_atoms_difference_norm": 0.008577510867520627,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9950508174988952,
-      "eval_loss": 0.30415305495262146,
+      "eval_linker_valid": 0.993813521873619,
+      "eval_loss": 0.2891532778739929,
       "eval_num_fragments": 3.0001767565178965,
-      "eval_poi_equal": 0.7625276182059213,
+      "eval_poi_equal": 0.6642509942554131,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9163941670349094,
-      "eval_poi_heavy_atoms_difference": 2.375784357048166,
-      "eval_poi_heavy_atoms_difference_norm": 0.07773493828827827,
+      "eval_poi_has_attachment_point(s)": 0.9058771542200619,
+      "eval_poi_heavy_atoms_difference": 3.015642951833849,
+      "eval_poi_heavy_atoms_difference_norm": 0.09752605319332988,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9163941670349094,
-      "eval_reassembly": 0.5579319487406098,
-      "eval_reassembly_nostereo": 0.5771984091913389,
-      "eval_runtime": 1499.4587,
-      "eval_samples_per_second": 7.546,
-      "eval_steps_per_second": 0.118,
+      "eval_poi_valid": 0.9058771542200619,
+      "eval_reassembly": 0.2889969067609368,
+      "eval_reassembly_nostereo": 0.30790985417587274,
+      "eval_runtime": 1566.8847,
+      "eval_samples_per_second": 7.221,
+      "eval_steps_per_second": 0.113,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.900309323906319,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.08187946677207947,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0039,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.09059714525938034,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0048,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.08514947444200516,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.003,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.07051639258861542,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0026,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.06919129192829132,
-      "learning_rate": 5.8123802212920094e-05,
-      "loss": 0.0026,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5527176314626602,
-      "eval_e3_equal": 0.8117543084401238,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9882456915598763,
-      "eval_e3_heavy_atoms_difference": 0.591427308882015,
-      "eval_e3_heavy_atoms_difference_norm": 0.01492656154342532,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9882456915598763,
-      "eval_has_all_attachment_points": 0.9825011047282368,
-      "eval_has_three_substructures": 0.9997348652231551,
-      "eval_heavy_atoms_difference": 8.222271321254972,
-      "eval_heavy_atoms_difference_norm": 0.10949622260200928,
-      "eval_linker_equal": 0.786389748121962,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9974370304904994,
-      "eval_linker_heavy_atoms_difference": 0.3985859478568272,
-      "eval_linker_heavy_atoms_difference_norm": 0.0064264088344804585,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9974370304904994,
-      "eval_loss": 0.31640464067459106,
-      "eval_num_fragments": 3.0000883782589485,
-      "eval_poi_equal": 0.7676535572249227,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9161290322580645,
-      "eval_poi_heavy_atoms_difference": 2.5651789659743702,
-      "eval_poi_heavy_atoms_difference_norm": 0.0863333744044112,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9161290322580645,
-      "eval_reassembly": 0.5608484312859037,
-      "eval_reassembly_nostereo": 0.583738400353513,
-      "eval_runtime": 1514.3528,
-      "eval_samples_per_second": 7.472,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9034909412284577,
-      "step": 10000
+      "eval_valid": 0.8980998674326116,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=23-learning_rate=5.8e-05-warmup_ratio=0.050-min_lr=0.000-factor=0.800",
+  "trial_name": "trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900",
   "trial_params": {
-    "factor": 0.8,
-    "learning_rate": 5.8123802212920094e-05,
-    "min_lr": 2.921430983672087e-10,
-    "warmup_ratio": 0.05
+    "factor": 0.9,
+    "learning_rate": 1.4902432220520327e-05,
+    "min_lr": 4.880860382192317e-09,
+    "warmup_ratio": 0.03
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:413b00f556ab5f243d0945f5dd0ccdffcf686846de60055d1be5a14ab331720e
+oid sha256:ea8467410704074ba62b10eb8dea889f4894df7f8372e47703ac4c704b91baf8
 size 7480