Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5784fcf8345c5f6ec3d4510e6917e3420b22eeeb4106250fb0e2417a5bec9920
+oid sha256:9b5738965b5bf849b4663b34cf7f2e202588eab86b05a91914031292c41bb808
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b5425990baa027defe8696b919751b73f689b94d73ac7d77c8746dda9088b7d
+oid sha256:0955baf75a00a2e88a7c1d335c5b797e63955ae2c94e71a66fcb5594f05f5205
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf2c0ab31f72811cb4f978b3735b946f859bcbc16feb41b3012458d6ad4bd376
+oid sha256:298531dcec7e369c7ae2614774e730ea732dc33482451d8049610850eb9dc84f
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d0f18d37de5901dacb817a2bed2d4e6c9e6637a57fb3719a7fbb81d9922296e
+oid sha256:8bbca288c354b7434ffbef97a3e61e0d47d7cef73a7df77bc8ca2c0bb4180bf9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.0007954043305346885,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2.000/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.05019885108263367,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2.000/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.695271763146266,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.5555965304374695,
+      "learning_rate": 8.9732158438651e-06,
+      "loss": 0.1821,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.6741116046905518,
+      "learning_rate": 7.42563181020302e-06,
+      "loss": 0.1498,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.43884631991386414,
+      "learning_rate": 4.6075656327911444e-06,
+      "loss": 0.1295,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.4529663026332855,
+      "learning_rate": 1.7568798478286516e-06,
+      "loss": 0.1177,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.3748651146888733,
+      "learning_rate": 1.257654662879903e-07,
+      "loss": 0.1123,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.01246133451171012,
+      "eval_e3_equal": 0.46548828988068935,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.997348652231551,
+      "eval_e3_heavy_atoms_difference": -0.7612019443216969,
+      "eval_e3_heavy_atoms_difference_norm": -0.04587739039853045,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.997348652231551,
+      "eval_has_all_attachment_points": 0.9699513919575784,
+      "eval_has_three_substructures": 0.9981440565620857,
+      "eval_heavy_atoms_difference": 12.529650905877155,
+      "eval_heavy_atoms_difference_norm": 0.16318795289666757,
+      "eval_linker_equal": 0.037560760053026956,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9932832523199293,
+      "eval_linker_heavy_atoms_difference": 2.51064958020327,
+      "eval_linker_heavy_atoms_difference_norm": 0.07505343220199563,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9932832523199293,
+      "eval_loss": 0.32810720801353455,
+      "eval_num_fragments": 2.998497569597879,
+      "eval_poi_equal": 0.4675209898365002,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.8659301811754309,
+      "eval_poi_heavy_atoms_difference": 4.305877154220062,
+      "eval_poi_heavy_atoms_difference_norm": 0.13126616905176186,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.8659301811754309,
+      "eval_reassembly": 0.012638091029606717,
+      "eval_reassembly_nostereo": 0.01732213875386655,
+      "eval_runtime": 1548.6457,
+      "eval_samples_per_second": 7.306,
+      "eval_steps_per_second": 0.114,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8623950508174989,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.44900745153427124,
+      "learning_rate": 4.307050893759296e-07,
+      "loss": 0.1117,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.4135892689228058,
+      "learning_rate": 2.537751069500334e-06,
+      "loss": 0.1106,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.5324644446372986,
+      "learning_rate": 5.5213632963006756e-06,
+      "loss": 0.1057,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.4086337983608246,
+      "learning_rate": 8.070961547174129e-06,
+      "loss": 0.097,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.4266679883003235,
+      "learning_rate": 9.066610391784642e-06,
+      "loss": 0.0854,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.05019885108263367,
+      "eval_e3_equal": 0.5632346442775077,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.997613787008396,
+      "eval_e3_heavy_atoms_difference": -0.43367211665930183,
+      "eval_e3_heavy_atoms_difference_norm": -0.03303006338880065,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.997613787008396,
+      "eval_has_all_attachment_points": 0.9743703049049933,
+      "eval_has_three_substructures": 0.9984975695978789,
+      "eval_heavy_atoms_difference": 10.36880247459125,
+      "eval_heavy_atoms_difference_norm": 0.13736942930541826,
+      "eval_linker_equal": 0.10234202386212991,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9954043305346885,
+      "eval_linker_heavy_atoms_difference": 1.9465311533362792,
+      "eval_linker_heavy_atoms_difference_norm": 0.06537832997851392,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9954043305346885,
+      "eval_loss": 0.3190831243991852,
+      "eval_num_fragments": 3.0000883782589485,
+      "eval_poi_equal": 0.5321254971277066,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.8882898806893504,
+      "eval_poi_heavy_atoms_difference": 3.66858152894388,
+      "eval_poi_heavy_atoms_difference_norm": 0.1148555997094906,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.8882898806893504,
+      "eval_reassembly": 0.05152452496685815,
+      "eval_reassembly_nostereo": 0.06310207688908528,
+      "eval_runtime": 1477.1687,
+      "eval_samples_per_second": 7.66,
+      "eval_steps_per_second": 0.12,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8853733981440566,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2.000",
   "trial_params": {