Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5145c2cc7d97ae3d9a124fc843bd98b3ed8f6f8784b198e6ff821ea49461e95
+oid sha256:c9c4ba6b0367d67c0c6a93ff9a6f9532bce9e8ab18d8a704566f81061f9bfe49
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7abd2543c0f4161003949ca3b0a09f32ee52189eb6cf1ce1b0d339218db56fb
+oid sha256:6890c852cd308a7eff18578bc70807f77e779ace9794b1916a6350a669080028
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56116cd4c05fa65ee5b68ac09ebd53a9cecc2c132b0229f70f547f950f24eb44
+oid sha256:3054464067f45429ad176605d6401de1dd7bd5705e90dadf9ec3f5b0e2d701e1
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8cec5d697af01fd16026abee650745e0dfef8562ef22eba2f073c03e205449e
+oid sha256:7c8036a0d2d386cb6b32ab93f67cfea9d25b9f22dfed89381daf4e334561e8a1
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=12-learning_rate=2.2e-06-warmup_ratio=0.050-num_cycles=10/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 8.837825894829872e-05,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=12-learning_rate=2.2e-06-warmup_ratio=0.050-num_cycles=10/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.24984533804684048,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 1.1844513416290283,
+      "learning_rate": 1.2467276257933323e-06,
+      "loss": 0.6296,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 1.2230936288833618,
+      "learning_rate": 8.78055397904762e-09,
+      "loss": 0.5823,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 1.0926300287246704,
+      "learning_rate": 1.2123738790757074e-06,
+      "loss": 0.5508,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 1.1617790460586548,
+      "learning_rate": 4.941964361068419e-09,
+      "loss": 0.5145,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 1.0982576608657837,
+      "learning_rate": 1.1778983265099774e-06,
+      "loss": 0.4877,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.09306230667255855,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9841802916482545,
+      "eval_e3_heavy_atoms_difference": -0.592311091471498,
+      "eval_e3_heavy_atoms_difference_norm": -0.05829268892151232,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9841802916482545,
+      "eval_has_all_attachment_points": 0.8215642951833849,
+      "eval_has_three_substructures": 0.9858594785682722,
+      "eval_heavy_atoms_difference": 53.983650022094565,
+      "eval_heavy_atoms_difference_norm": 0.6927416972140691,
+      "eval_linker_equal": 0.0019443216968625717,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9680070702607159,
+      "eval_linker_heavy_atoms_difference": 7.140874944763588,
+      "eval_linker_heavy_atoms_difference_norm": 0.31848802047032876,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9680070702607159,
+      "eval_loss": 0.570651650428772,
+      "eval_num_fragments": 3.0021210782147594,
+      "eval_poi_equal": 0.012196199734865224,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.3939019001325674,
+      "eval_poi_heavy_atoms_difference": 22.490499337163058,
+      "eval_poi_heavy_atoms_difference_norm": 0.6740630548863725,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.3939019001325674,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 1404.1284,
+      "eval_samples_per_second": 8.058,
+      "eval_steps_per_second": 0.126,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.38700839593460007,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 1.0194402933120728,
+      "learning_rate": 2.1973504834757866e-09,
+      "loss": 0.4597,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 1.184683918952942,
+      "learning_rate": 1.1433356820552028e-06,
+      "loss": 0.4365,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.9286699295043945,
+      "learning_rate": 5.494759397925871e-10,
+      "loss": 0.413,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 1.2059142589569092,
+      "learning_rate": 1.108720747364596e-06,
+      "loss": 0.394,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.8440704941749573,
+      "learning_rate": 0.0,
+      "loss": 0.3739,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 8.837825894829872e-05,
+      "eval_e3_equal": 0.11904551480335837,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9881573133009279,
+      "eval_e3_heavy_atoms_difference": -1.3612903225806452,
+      "eval_e3_heavy_atoms_difference_norm": -0.08243470960376242,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9881573133009279,
+      "eval_has_all_attachment_points": 0.8463985859478568,
+      "eval_has_three_substructures": 0.9893946089262041,
+      "eval_heavy_atoms_difference": 44.24374723817941,
+      "eval_heavy_atoms_difference_norm": 0.5653104509128849,
+      "eval_linker_equal": 0.0022978347326557667,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9764030048608042,
+      "eval_linker_heavy_atoms_difference": 6.987450287229342,
+      "eval_linker_heavy_atoms_difference_norm": 0.3159726484448598,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9764030048608042,
+      "eval_loss": 0.4938018321990967,
+      "eval_num_fragments": 2.997348652231551,
+      "eval_poi_equal": 0.03623508616880247,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.5198409191338931,
+      "eval_poi_heavy_atoms_difference": 18.02191780821918,
+      "eval_poi_heavy_atoms_difference_norm": 0.526393787824688,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.5198409191338931,
+      "eval_reassembly": 8.837825894829872e-05,
+      "eval_reassembly_nostereo": 8.837825894829872e-05,
+      "eval_runtime": 1451.854,
+      "eval_samples_per_second": 7.793,
+      "eval_steps_per_second": 0.122,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.5143614670790986,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=12-learning_rate=2.2e-06-warmup_ratio=0.050-num_cycles=10",
   "trial_params": {