Training in progress, step 20000, checkpoint

last-checkpoint/generation_config.json

@@ -1,4 +1,5 @@
 {
+  "_from_model_config": true,
   "bos_token_id": 0,
   "do_sample": true,
   "eos_token_id": 2,

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a1a533f600d03176bbc433be1da1958ca69bf5a64c3a1b518e0290ba9ec84d5
+oid sha256:31bdc100ec33c5cc9ceb7d25df38c6bc73ccb1bf21fe4779b8c41687ea533d99
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d230c26c55815659f63e169d0f9a46f10653a4cf9b847c53d32149df09fb219
+oid sha256:83043627f0b085b535e0816fe564fca74eaa0f7b1f30ea74a5e0862f180c25a1
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:919d51343ba7f79f11689f549f310632a28ac5ec6681d1245582e8580ebcf5fc
+oid sha256:b53e134a457f989a320cac37a5c1901d5b22cc7775ac770c8c1f5df456090d0c
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a6fdd0313393abbe7489aa1bcb1f52fb3950b20e996b10a6e64befc91a1cdcd
+oid sha256:ec08642f74697f111f64e755e2fc9bd211b07cc951ee155be7841c13ee3a6fae
 size 1192

last-checkpoint/trainer_state.json

@@ -1,285 +1,47 @@
 {
-  "best_metric": 0.4881131241714538,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900/checkpoint-10000",
-  "epoch": 0.9863878477017163,
-  "eval_steps": 2500,
-  "global_step": 10000,
-  "is_hyper_param_search": true,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9727756954034326,
+  "eval_steps": 20000,
+  "global_step": 20000,
+  "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.049319392385085814,
-      "grad_norm": 1.0362313985824585,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 1.5088,
-      "step": 500
-    },
-    {
-      "epoch": 0.09863878477017163,
-      "grad_norm": 0.7315112948417664,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.4704,
-      "step": 1000
-    },
-    {
-      "epoch": 0.14795817715525744,
-      "grad_norm": 0.7089930176734924,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.2658,
-      "step": 1500
-    },
-    {
-      "epoch": 0.19727756954034326,
-      "grad_norm": 0.517996609210968,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.1763,
-      "step": 2000
-    },
-    {
-      "epoch": 0.24659696192542907,
-      "grad_norm": 0.5622127056121826,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.1252,
-      "step": 2500
-    },
-    {
-      "epoch": 0.2959163543105149,
-      "grad_norm": 0.438522070646286,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.092,
-      "step": 3000
-    },
-    {
-      "epoch": 0.3452357466956007,
-      "grad_norm": 0.3132740259170532,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0692,
-      "step": 3500
-    },
-    {
-      "epoch": 0.3945551390806865,
-      "grad_norm": 0.3049832582473755,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0546,
-      "step": 4000
-    },
-    {
-      "epoch": 0.4438745314657723,
-      "grad_norm": 0.32669779658317566,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0438,
-      "step": 4500
-    },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.34408244490623474,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0364,
+      "grad_norm": 0.15789619088172913,
+      "learning_rate": 5e-05,
+      "loss": 0.1302,
       "step": 5000
     },
-    {
-      "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.2837825894829872,
-      "eval_e3_equal": 0.747326557666814,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9939019001325674,
-      "eval_e3_heavy_atoms_difference": 0.1380468404772426,
-      "eval_e3_heavy_atoms_difference_norm": -0.004540464395693658,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9939019001325674,
-      "eval_has_all_attachment_points": 0.9800265134776844,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 9.344498453380469,
-      "eval_heavy_atoms_difference_norm": 0.12324719766631663,
-      "eval_linker_equal": 0.4260715863897481,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.993813521873619,
-      "eval_linker_heavy_atoms_difference": 1.339637649138312,
-      "eval_linker_heavy_atoms_difference_norm": 0.04383051599945908,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.993813521873619,
-      "eval_loss": 0.2891532778739929,
-      "eval_num_fragments": 3.0001767565178965,
-      "eval_poi_equal": 0.6642509942554131,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9058771542200619,
-      "eval_poi_heavy_atoms_difference": 3.015642951833849,
-      "eval_poi_heavy_atoms_difference_norm": 0.09752605319332988,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9058771542200619,
-      "eval_reassembly": 0.2889969067609368,
-      "eval_reassembly_nostereo": 0.30790985417587274,
-      "eval_runtime": 1566.8847,
-      "eval_samples_per_second": 7.221,
-      "eval_steps_per_second": 0.113,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8980998674326116,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.3127955496311188,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0303,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.24419963359832764,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0256,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.2199561446905136,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0218,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.2431618571281433,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0188,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.18661023676395416,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0163,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.43747238179407866,
-      "eval_e3_equal": 0.7806451612903226,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9870967741935484,
-      "eval_e3_heavy_atoms_difference": 0.3549270879363677,
-      "eval_e3_heavy_atoms_difference_norm": 0.004763396121020274,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9870967741935484,
-      "eval_has_all_attachment_points": 0.9794962439239947,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 7.966681396376491,
-      "eval_heavy_atoms_difference_norm": 0.10506341820964833,
-      "eval_linker_equal": 0.628104286345559,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9942554131683606,
-      "eval_linker_heavy_atoms_difference": 0.5155987627043748,
-      "eval_linker_heavy_atoms_difference_norm": 0.004264745639607716,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9942554131683606,
-      "eval_loss": 0.29560938477516174,
-      "eval_num_fragments": 3.0007070260715865,
-      "eval_poi_equal": 0.7138311975254088,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9196641626159965,
-      "eval_poi_heavy_atoms_difference": 2.46336721166593,
-      "eval_poi_heavy_atoms_difference_norm": 0.07938490626627487,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9196641626159965,
-      "eval_reassembly": 0.44410075121520104,
-      "eval_reassembly_nostereo": 0.46796288113124174,
-      "eval_runtime": 1492.3352,
-      "eval_samples_per_second": 7.582,
-      "eval_steps_per_second": 0.119,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.905346884666372,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2039629966020584,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0146,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.22314690053462982,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.013,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.23112636804580688,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0115,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.24067600071430206,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0105,
-      "step": 9500
-    },
     {
       "epoch": 0.9863878477017163,
-      "grad_norm": 0.22660063207149506,
-      "learning_rate": 1.4902432220520327e-05,
-      "loss": 0.0097,
+      "grad_norm": 0.08171849697828293,
+      "learning_rate": 5e-05,
+      "loss": 0.0042,
       "step": 10000
     },
     {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.4881131241714538,
-      "eval_e3_equal": 0.7934600088378259,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.993813521873619,
-      "eval_e3_heavy_atoms_difference": 0.2950066283694211,
-      "eval_e3_heavy_atoms_difference_norm": 0.0028168010095192407,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.993813521873619,
-      "eval_has_all_attachment_points": 0.9835616438356164,
-      "eval_has_three_substructures": 0.9992045956694653,
-      "eval_heavy_atoms_difference": 7.484577993813522,
-      "eval_heavy_atoms_difference_norm": 0.0982968303528513,
-      "eval_linker_equal": 0.6990720282810429,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9962881131241714,
-      "eval_linker_heavy_atoms_difference": 0.5271763146266019,
-      "eval_linker_heavy_atoms_difference_norm": 0.010195950921219643,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9962881131241714,
-      "eval_loss": 0.3071575164794922,
-      "eval_num_fragments": 3.000265134776845,
-      "eval_poi_equal": 0.73194874060981,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.922757401679187,
-      "eval_poi_heavy_atoms_difference": 2.526292532037119,
-      "eval_poi_heavy_atoms_difference_norm": 0.08038544461395786,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.922757401679187,
-      "eval_reassembly": 0.49597878921785243,
-      "eval_reassembly_nostereo": 0.5214317277949624,
-      "eval_runtime": 1469.8755,
-      "eval_samples_per_second": 7.698,
-      "eval_steps_per_second": 0.12,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9151568714096332,
-      "step": 10000
+      "epoch": 1.4795817715525745,
+      "grad_norm": 0.07135193794965744,
+      "learning_rate": 5e-05,
+      "loss": 0.0023,
+      "step": 15000
+    },
+    {
+      "epoch": 1.9727756954034326,
+      "grad_norm": 0.03533194214105606,
+      "learning_rate": 5e-05,
+      "loss": 0.0017,
+      "step": 20000
     }
   ],
-  "logging_steps": 500,
-  "max_steps": 10000,
+  "logging_steps": 5000,
+  "max_steps": 100000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
-  "save_steps": 5000,
+  "num_train_epochs": 10,
+  "save_steps": 20000,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -287,18 +49,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 1.7328802956260813e+17,
   "train_batch_size": 128,
-  "trial_name": "trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900",
-  "trial_params": {
-    "factor": 0.9,
-    "learning_rate": 1.4902432220520327e-05,
-    "min_lr": 4.880860382192317e-09,
-    "warmup_ratio": 0.03
-  }
+  "trial_name": null,
+  "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea8467410704074ba62b10eb8dea889f4894df7f8372e47703ac4c704b91baf8
-size 7480
+oid sha256:dcb7bfc978254f0312b2a392a15fc5cd00c5803ad7cc88491d0b6f6dd53c348b
+size 7416