Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7858c315818de5130acbbc619c1eac6686eddaced96014ed44abf64bd08f34e9
+oid sha256:bb40afa0f98eeba2978ead614c940f65454c8f8f162db89b34209504b74a895c
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac4db3335bf9d6e020fd2656b9fd8ff7e8da2cd70247ef61a2b820090afd4784
+oid sha256:b13b475fd10eec41713f5d112fba0bf80de8a2f9691159a43f9e74ecfa2b18fc
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0acce1ad3c5266197d8115b92421ffd2ee6dfe8d6ff670056ad51fa28e27cf0d
+oid sha256:0064953f8e361e9b3b607021419c020a9c1a3b9a33a36c311ffc54273cce7c39
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbe1b98503cbd4292623e348d08bd4c2378afec44628d482987993cb89691b96
+oid sha256:8d74e6a70fffafc0ee8caf5f7183d8b7c8c69e9bd6343cd7115a7c7d9a909424
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.10225364560318162,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=9/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.4198851082633672,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=9/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8852850198851082,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.2906638979911804,
+      "learning_rate": 3.9773040302723844e-07,
+      "loss": 0.0562,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.35333454608917236,
+      "learning_rate": 1.3897677361316137e-05,
+      "loss": 0.0524,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.30168837308883667,
+      "learning_rate": 1.558699860249444e-06,
+      "loss": 0.0417,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.2775203287601471,
+      "learning_rate": 1.6248747079137917e-05,
+      "loss": 0.039,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.2152932584285736,
+      "learning_rate": 3.3888535307937097e-06,
+      "loss": 0.0321,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.311533362792753,
+      "eval_e3_equal": 0.743791427308882,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9951391957578436,
+      "eval_e3_heavy_atoms_difference": 0.0415377817057004,
+      "eval_e3_heavy_atoms_difference_norm": -0.01008792799382512,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9951391957578436,
+      "eval_has_all_attachment_points": 0.9878921785240831,
+      "eval_has_three_substructures": 0.999116217410517,
+      "eval_heavy_atoms_difference": 7.704197967300044,
+      "eval_heavy_atoms_difference_norm": 0.10313429571149772,
+      "eval_linker_equal": 0.4563853292090146,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9963764913831198,
+      "eval_linker_heavy_atoms_difference": 0.8818382677861246,
+      "eval_linker_heavy_atoms_difference_norm": 0.02104385647512381,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9963764913831198,
+      "eval_loss": 0.2895846664905548,
+      "eval_num_fragments": 3.0001767565178965,
+      "eval_poi_equal": 0.6785682722050376,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9140963323022536,
+      "eval_poi_heavy_atoms_difference": 2.4948298718515245,
+      "eval_poi_heavy_atoms_difference_norm": 0.07935383938917676,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9140963323022536,
+      "eval_reassembly": 0.3170128148475475,
+      "eval_reassembly_nostereo": 0.34034467520989836,
+      "eval_runtime": 1552.5014,
+      "eval_samples_per_second": 7.288,
+      "eval_steps_per_second": 0.114,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9080866106937693,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.3061818480491638,
+      "learning_rate": 1.8078900749682204e-05,
+      "loss": 0.0294,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.2009110301733017,
+      "learning_rate": 5.73992324861551e-06,
+      "loss": 0.0262,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.3278745114803314,
+      "learning_rate": 1.923987020690441e-05,
+      "loss": 0.0223,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.20894533395767212,
+      "learning_rate": 8.421439327526943e-06,
+      "loss": 0.0216,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.2186155617237091,
+      "learning_rate": 0.0,
+      "loss": 0.0174,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.4198851082633672,
+      "eval_e3_equal": 0.7729562527618206,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9948740609809986,
+      "eval_e3_heavy_atoms_difference": 0.13380468404772425,
+      "eval_e3_heavy_atoms_difference_norm": -0.004228343690094682,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9948740609809986,
+      "eval_has_all_attachment_points": 0.9847989394608926,
+      "eval_has_three_substructures": 0.9995581087052585,
+      "eval_heavy_atoms_difference": 6.57870083959346,
+      "eval_heavy_atoms_difference_norm": 0.08838262204030779,
+      "eval_linker_equal": 0.5939902783915156,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.997348652231551,
+      "eval_linker_heavy_atoms_difference": 0.4752098983650022,
+      "eval_linker_heavy_atoms_difference_norm": -0.0007188213382745067,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.997348652231551,
+      "eval_loss": 0.29139745235443115,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.7161290322580646,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9248784798939461,
+      "eval_poi_heavy_atoms_difference": 2.1780821917808217,
+      "eval_poi_heavy_atoms_difference_norm": 0.0701177673293958,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9248784798939461,
+      "eval_reassembly": 0.4266902342023862,
+      "eval_reassembly_nostereo": 0.4508174988952718,
+      "eval_runtime": 1504.9536,
+      "eval_samples_per_second": 7.519,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9188687582854618,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=1-learning_rate=2.0e-05-warmup_ratio=0.090-num_cycles=9",
   "trial_params": {