Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6ddc8f2a3677e5e317b80ecd5acf9d1491b0668d5f53b8d9bd9e0d1abbfa2b5
+oid sha256:c6a5a578f0c1654cb1728c47533deb3ca07393d182847dbda9810c87d4eddda0
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d76ad6ce78a4b0037233c4839e647df1579c5d586d5c6854be18638b73cb4cb
+oid sha256:99e1a3594eb5da26b473ff86dcb3292d6312354cab75b099645fa4dc3938a31e
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c85698974fe6e7526ea4561e96c9ff34251353ffb22a51e97eb7da83eed5ae17
+oid sha256:78b719d256ecfb1a83a26593745fb074c4c974f8a70967bada23bf0b33d7fd2d
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db54f45be63f2a31cb673e60b81aae6b0e3b4b54ab7baae14cb41e91340d7eb8
+oid sha256:24e04b6917ab55f6326e4cf96da9b4fd749ccb1f6bee2620e7d3dd1a4888fb83
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.41140079540433055,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5699513919575785,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8162615996464869,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.0758630558848381,
+      "learning_rate": 2.739995113403538e-05,
+      "loss": 0.0125,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.10389214009046555,
+      "learning_rate": 0.00017844822299033737,
+      "loss": 0.0032,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.07643450051546097,
+      "learning_rate": 7.810415658232181e-05,
+      "loss": 0.0109,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.07314460724592209,
+      "learning_rate": 0.00012044362176156312,
+      "loss": 0.0027,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.07255467027425766,
+      "learning_rate": 0.00013918910789064985,
+      "loss": 0.0073,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.5348652231551039,
+      "eval_e3_equal": 0.8077772867874503,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9785240830755634,
+      "eval_e3_heavy_atoms_difference": 0.9194874060980999,
+      "eval_e3_heavy_atoms_difference_norm": 0.02738403050962625,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9785240830755634,
+      "eval_has_all_attachment_points": 0.9831197525408749,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 9.151480335837384,
+      "eval_heavy_atoms_difference_norm": 0.12145187956226944,
+      "eval_linker_equal": 0.7649138311975254,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9933716305788776,
+      "eval_linker_heavy_atoms_difference": 0.40450729120636325,
+      "eval_linker_heavy_atoms_difference_norm": 0.003936525020270963,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9933716305788776,
+      "eval_loss": 0.2837851345539093,
+      "eval_num_fragments": 2.999381352187362,
+      "eval_poi_equal": 0.750861688024746,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9107379584622183,
+      "eval_poi_heavy_atoms_difference": 2.652938577110031,
+      "eval_poi_heavy_atoms_difference_norm": 0.08625993996859925,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9107379584622183,
+      "eval_reassembly": 0.5424657534246575,
+      "eval_reassembly_nostereo": 0.562704374723818,
+      "eval_runtime": 1534.4399,
+      "eval_samples_per_second": 7.374,
+      "eval_steps_per_second": 0.115,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.887582854617764,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.06594572961330414,
+      "learning_rate": 6.0722472746074165e-05,
+      "loss": 0.0025,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.1494133025407791,
+      "learning_rate": 0.0001932015238178336,
+      "loss": 0.0095,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.040824271738529205,
+      "learning_rate": 1.634838960794289e-05,
+      "loss": 0.0045,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.21492616832256317,
+      "learning_rate": 0.00022470889823098504,
+      "loss": 0.0041,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.057117585092782974,
+      "learning_rate": 0.0,
+      "loss": 0.0062,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5699513919575785,
+      "eval_e3_equal": 0.8192664604507292,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9812638091029606,
+      "eval_e3_heavy_atoms_difference": 0.6555899248784799,
+      "eval_e3_heavy_atoms_difference_norm": 0.01675108799959528,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9812638091029606,
+      "eval_has_all_attachment_points": 0.989041095890411,
+      "eval_has_three_substructures": 0.9998232434821034,
+      "eval_heavy_atoms_difference": 7.600265134776845,
+      "eval_heavy_atoms_difference_norm": 0.10056611800605245,
+      "eval_linker_equal": 0.8079540433053469,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9977905435262925,
+      "eval_linker_heavy_atoms_difference": 0.27220503756076003,
+      "eval_linker_heavy_atoms_difference_norm": -0.002432511049529868,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9977905435262925,
+      "eval_loss": 0.26366686820983887,
+      "eval_num_fragments": 2.9998232434821035,
+      "eval_poi_equal": 0.7761378700839593,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9255855059655325,
+      "eval_poi_heavy_atoms_difference": 2.2732655766681398,
+      "eval_poi_heavy_atoms_difference_norm": 0.07230797500159608,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9255855059655325,
+      "eval_reassembly": 0.57870083959346,
+      "eval_reassembly_nostereo": 0.6014140521431728,
+      "eval_runtime": 1536.0555,
+      "eval_samples_per_second": 7.366,
+      "eval_steps_per_second": 0.115,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9069376933274415,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=15-learning_rate=2.3e-04-warmup_ratio=0.070-num_cycles=8.500",
   "trial_params": {