Training in progress, epoch 1, checkpoint

last-checkpoint/config.json

@@ -1,21 +1,30 @@
 {
-  "_name_or_path": "tattabio/gLM2_650M",
+  "_name_or_path": "facebook/esm2_t33_650M_UR50D",
   "architectures": [
-    "gLM2ForMaskedLM"
+    "EsmForMaskedLM"
   ],
-  "auto_map": {
-    "AutoConfig": "configuration_glm2.gLM2Config",
-    "AutoModel": "modeling_glm2.gLM2Model",
-    "AutoModelForMaskedLM": "modeling_glm2.gLM2ForMaskedLM"
-  },
-  "depth": 33,
-  "dim": 1280,
-  "ffn_dim_multiplier": null,
-  "heads": 20,
-  "model_type": "gLM2",
-  "norm_eps": 1e-05,
-  "swiglu_multiple_of": 256,
+  "attention_probs_dropout_prob": 0.0,
+  "classifier_dropout": null,
+  "emb_layer_norm_before": false,
+  "esmfold_config": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 1280,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "is_folding_model": false,
+  "layer_norm_eps": 1e-05,
+  "mask_token_id": 32,
+  "max_position_embeddings": 1026,
+  "model_type": "esm",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 33,
+  "pad_token_id": 1,
+  "position_embedding_type": "rotary",
+  "token_dropout": true,
   "torch_dtype": "float32",
   "transformers_version": "4.49.0",
-  "vocab_size": 37
+  "use_cache": true,
+  "vocab_list": null,
+  "vocab_size": 33
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60a8151bee68255d9064bbfdc2170764612ef8d251c1027fa3d7f12321916dbc
-size 2682482800
+oid sha256:414be13553d235730cd6c247e44cbd6a06aedc9a3e32ed0e9d4ae9d408220e05
+size 2609498088

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40239a22d776e1fb8b4210a19644be230adf89378f030068f6bdc92cdbebfd01
-size 5365108834
+oid sha256:98f0ac03d2d0d85353d96c13a42f8dd4e6648b6dc703fec691b45435eb8c1437
+size 5208796146

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67e8fb856d223a5af24cc75d2aa8b4de37cdfc3cbf75d495b03ac0cbca8dbef4
+oid sha256:d880ce44acc3bc8f93d20fb478a852664f88c91d3c7c6a2fac143962de832a8b
 size 15006

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c15dd75d8ac97bb6e7d4107e91cba13385d6d6961fc51bf55911773daaa9d375
+oid sha256:bca4dbe650e04bc8012dd3f1938dfb2a637329721abd75c3bd59d28a64007b54
 size 1064

last-checkpoint/trainer_state.json

@@ -1,466 +1,46 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.997206530510894,
+  "epoch": 1.0,
   "eval_steps": 50,
-  "global_step": 1506,
+  "global_step": 129,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.1986467192252778,
-      "grad_norm": 1.6958907842636108,
-      "learning_rate": 9.800796812749005e-05,
-      "loss": 1.6362,
+      "epoch": 0.3904343582235237,
+      "grad_norm": 0.5675944685935974,
+      "learning_rate": 9.609375e-05,
+      "loss": 1.5678,
       "step": 50
     },
     {
-      "epoch": 0.1986467192252778,
-      "eval_loss": 1.5567175149917603,
-      "eval_runtime": 14.8041,
-      "eval_samples_per_second": 57.281,
-      "eval_steps_per_second": 28.641,
+      "epoch": 0.3904343582235237,
+      "eval_loss": 1.53541898727417,
+      "eval_runtime": 11.6265,
+      "eval_samples_per_second": 37.156,
+      "eval_steps_per_second": 18.578,
       "step": 50
     },
     {
-      "epoch": 0.3972934384505556,
-      "grad_norm": 1.6160608530044556,
-      "learning_rate": 9.601593625498009e-05,
-      "loss": 1.5283,
+      "epoch": 0.7808687164470474,
+      "grad_norm": 0.5330150127410889,
+      "learning_rate": 9.21875e-05,
+      "loss": 1.5019,
       "step": 100
     },
     {
-      "epoch": 0.3972934384505556,
-      "eval_loss": 1.5002830028533936,
-      "eval_runtime": 14.7266,
-      "eval_samples_per_second": 57.583,
-      "eval_steps_per_second": 28.792,
+      "epoch": 0.7808687164470474,
+      "eval_loss": 1.4973269701004028,
+      "eval_runtime": 11.5507,
+      "eval_samples_per_second": 37.4,
+      "eval_steps_per_second": 18.7,
       "step": 100
-    },
-    {
-      "epoch": 0.5959401576758334,
-      "grad_norm": 1.500954270362854,
-      "learning_rate": 9.402390438247013e-05,
-      "loss": 1.4825,
-      "step": 150
-    },
-    {
-      "epoch": 0.5959401576758334,
-      "eval_loss": 1.4542008638381958,
-      "eval_runtime": 14.5669,
-      "eval_samples_per_second": 58.214,
-      "eval_steps_per_second": 29.107,
-      "step": 150
-    },
-    {
-      "epoch": 0.7945868769011112,
-      "grad_norm": 0.8321912884712219,
-      "learning_rate": 9.203187250996016e-05,
-      "loss": 1.4431,
-      "step": 200
-    },
-    {
-      "epoch": 0.7945868769011112,
-      "eval_loss": 1.4306951761245728,
-      "eval_runtime": 14.563,
-      "eval_samples_per_second": 58.23,
-      "eval_steps_per_second": 29.115,
-      "step": 200
-    },
-    {
-      "epoch": 0.993233596126389,
-      "grad_norm": 1.2672511339187622,
-      "learning_rate": 9.00398406374502e-05,
-      "loss": 1.4083,
-      "step": 250
-    },
-    {
-      "epoch": 0.993233596126389,
-      "eval_loss": 1.3854182958602905,
-      "eval_runtime": 14.6247,
-      "eval_samples_per_second": 57.984,
-      "eval_steps_per_second": 28.992,
-      "step": 250
-    },
-    {
-      "epoch": 1.1946737848407722,
-      "grad_norm": 1.051540493965149,
-      "learning_rate": 8.804780876494024e-05,
-      "loss": 1.3707,
-      "step": 300
-    },
-    {
-      "epoch": 1.1946737848407722,
-      "eval_loss": 1.3773671388626099,
-      "eval_runtime": 14.6834,
-      "eval_samples_per_second": 57.752,
-      "eval_steps_per_second": 28.876,
-      "step": 300
-    },
-    {
-      "epoch": 1.39332050406605,
-      "grad_norm": 0.9810565710067749,
-      "learning_rate": 8.605577689243029e-05,
-      "loss": 1.331,
-      "step": 350
-    },
-    {
-      "epoch": 1.39332050406605,
-      "eval_loss": 1.33962881565094,
-      "eval_runtime": 14.6332,
-      "eval_samples_per_second": 57.95,
-      "eval_steps_per_second": 28.975,
-      "step": 350
-    },
-    {
-      "epoch": 1.5919672232913278,
-      "grad_norm": 1.0661150217056274,
-      "learning_rate": 8.406374501992032e-05,
-      "loss": 1.314,
-      "step": 400
-    },
-    {
-      "epoch": 1.5919672232913278,
-      "eval_loss": 1.3049601316452026,
-      "eval_runtime": 14.7683,
-      "eval_samples_per_second": 57.42,
-      "eval_steps_per_second": 28.71,
-      "step": 400
-    },
-    {
-      "epoch": 1.7906139425166057,
-      "grad_norm": 1.1233532428741455,
-      "learning_rate": 8.207171314741037e-05,
-      "loss": 1.2799,
-      "step": 450
-    },
-    {
-      "epoch": 1.7906139425166057,
-      "eval_loss": 1.2853434085845947,
-      "eval_runtime": 14.6881,
-      "eval_samples_per_second": 57.734,
-      "eval_steps_per_second": 28.867,
-      "step": 450
-    },
-    {
-      "epoch": 1.9892606617418833,
-      "grad_norm": 0.9072484970092773,
-      "learning_rate": 8.00796812749004e-05,
-      "loss": 1.2684,
-      "step": 500
-    },
-    {
-      "epoch": 1.9892606617418833,
-      "eval_loss": 1.2741819620132446,
-      "eval_runtime": 14.6458,
-      "eval_samples_per_second": 57.901,
-      "eval_steps_per_second": 28.95,
-      "step": 500
-    },
-    {
-      "epoch": 2.190700850456267,
-      "grad_norm": 1.057379961013794,
-      "learning_rate": 7.808764940239044e-05,
-      "loss": 1.241,
-      "step": 550
-    },
-    {
-      "epoch": 2.190700850456267,
-      "eval_loss": 1.25983726978302,
-      "eval_runtime": 14.6655,
-      "eval_samples_per_second": 57.823,
-      "eval_steps_per_second": 28.911,
-      "step": 550
-    },
-    {
-      "epoch": 2.3893475696815445,
-      "grad_norm": 1.0341771841049194,
-      "learning_rate": 7.609561752988048e-05,
-      "loss": 1.2102,
-      "step": 600
-    },
-    {
-      "epoch": 2.3893475696815445,
-      "eval_loss": 1.2425962686538696,
-      "eval_runtime": 14.6716,
-      "eval_samples_per_second": 57.799,
-      "eval_steps_per_second": 28.899,
-      "step": 600
-    },
-    {
-      "epoch": 2.587994288906822,
-      "grad_norm": 0.9421936869621277,
-      "learning_rate": 7.410358565737052e-05,
-      "loss": 1.2018,
-      "step": 650
-    },
-    {
-      "epoch": 2.587994288906822,
-      "eval_loss": 1.2194551229476929,
-      "eval_runtime": 14.6762,
-      "eval_samples_per_second": 57.781,
-      "eval_steps_per_second": 28.89,
-      "step": 650
-    },
-    {
-      "epoch": 2.7866410081321,
-      "grad_norm": 1.0019429922103882,
-      "learning_rate": 7.211155378486057e-05,
-      "loss": 1.1846,
-      "step": 700
-    },
-    {
-      "epoch": 2.7866410081321,
-      "eval_loss": 1.2068556547164917,
-      "eval_runtime": 14.7447,
-      "eval_samples_per_second": 57.512,
-      "eval_steps_per_second": 28.756,
-      "step": 700
-    },
-    {
-      "epoch": 2.985287727357378,
-      "grad_norm": 1.0412020683288574,
-      "learning_rate": 7.01195219123506e-05,
-      "loss": 1.1678,
-      "step": 750
-    },
-    {
-      "epoch": 2.985287727357378,
-      "eval_loss": 1.199570655822754,
-      "eval_runtime": 17.8039,
-      "eval_samples_per_second": 47.63,
-      "eval_steps_per_second": 23.815,
-      "step": 750
-    },
-    {
-      "epoch": 3.186727916071761,
-      "grad_norm": 0.9792215824127197,
-      "learning_rate": 6.812749003984064e-05,
-      "loss": 1.1527,
-      "step": 800
-    },
-    {
-      "epoch": 3.186727916071761,
-      "eval_loss": 1.1857693195343018,
-      "eval_runtime": 14.7604,
-      "eval_samples_per_second": 57.451,
-      "eval_steps_per_second": 28.726,
-      "step": 800
-    },
-    {
-      "epoch": 3.385374635297039,
-      "grad_norm": 0.916307806968689,
-      "learning_rate": 6.613545816733068e-05,
-      "loss": 1.1294,
-      "step": 850
-    },
-    {
-      "epoch": 3.385374635297039,
-      "eval_loss": 1.1673567295074463,
-      "eval_runtime": 32.8585,
-      "eval_samples_per_second": 25.808,
-      "eval_steps_per_second": 12.904,
-      "step": 850
-    },
-    {
-      "epoch": 3.5840213545223167,
-      "grad_norm": 0.9643361568450928,
-      "learning_rate": 6.414342629482072e-05,
-      "loss": 1.1162,
-      "step": 900
-    },
-    {
-      "epoch": 3.5840213545223167,
-      "eval_loss": 1.1727643013000488,
-      "eval_runtime": 14.7175,
-      "eval_samples_per_second": 57.619,
-      "eval_steps_per_second": 28.809,
-      "step": 900
-    },
-    {
-      "epoch": 3.7826680737475944,
-      "grad_norm": 0.9754778146743774,
-      "learning_rate": 6.215139442231077e-05,
-      "loss": 1.1016,
-      "step": 950
-    },
-    {
-      "epoch": 3.7826680737475944,
-      "eval_loss": 1.1499500274658203,
-      "eval_runtime": 14.7384,
-      "eval_samples_per_second": 57.537,
-      "eval_steps_per_second": 28.768,
-      "step": 950
-    },
-    {
-      "epoch": 3.9813147929728725,
-      "grad_norm": 0.9538551568984985,
-      "learning_rate": 6.01593625498008e-05,
-      "loss": 1.0814,
-      "step": 1000
-    },
-    {
-      "epoch": 3.9813147929728725,
-      "eval_loss": 1.1356687545776367,
-      "eval_runtime": 14.7227,
-      "eval_samples_per_second": 57.598,
-      "eval_steps_per_second": 28.799,
-      "step": 1000
-    },
-    {
-      "epoch": 4.182754981687255,
-      "grad_norm": 0.9160069227218628,
-      "learning_rate": 5.816733067729084e-05,
-      "loss": 1.0749,
-      "step": 1050
-    },
-    {
-      "epoch": 4.182754981687255,
-      "eval_loss": 1.1225874423980713,
-      "eval_runtime": 14.7411,
-      "eval_samples_per_second": 57.526,
-      "eval_steps_per_second": 28.763,
-      "step": 1050
-    },
-    {
-      "epoch": 4.381401700912534,
-      "grad_norm": 1.1243151426315308,
-      "learning_rate": 5.6175298804780876e-05,
-      "loss": 1.0462,
-      "step": 1100
-    },
-    {
-      "epoch": 4.381401700912534,
-      "eval_loss": 1.1159089803695679,
-      "eval_runtime": 14.5859,
-      "eval_samples_per_second": 58.138,
-      "eval_steps_per_second": 29.069,
-      "step": 1100
-    },
-    {
-      "epoch": 4.580048420137811,
-      "grad_norm": 1.018583059310913,
-      "learning_rate": 5.418326693227092e-05,
-      "loss": 1.052,
-      "step": 1150
-    },
-    {
-      "epoch": 4.580048420137811,
-      "eval_loss": 1.1180405616760254,
-      "eval_runtime": 14.8148,
-      "eval_samples_per_second": 57.24,
-      "eval_steps_per_second": 28.62,
-      "step": 1150
-    },
-    {
-      "epoch": 4.778695139363089,
-      "grad_norm": 0.9607245922088623,
-      "learning_rate": 5.219123505976096e-05,
-      "loss": 1.0432,
-      "step": 1200
-    },
-    {
-      "epoch": 4.778695139363089,
-      "eval_loss": 1.0965369939804077,
-      "eval_runtime": 14.6889,
-      "eval_samples_per_second": 57.731,
-      "eval_steps_per_second": 28.865,
-      "step": 1200
-    },
-    {
-      "epoch": 4.977341858588367,
-      "grad_norm": 1.06922447681427,
-      "learning_rate": 5.0199203187251e-05,
-      "loss": 1.0289,
-      "step": 1250
-    },
-    {
-      "epoch": 4.977341858588367,
-      "eval_loss": 1.1008275747299194,
-      "eval_runtime": 14.6242,
-      "eval_samples_per_second": 57.986,
-      "eval_steps_per_second": 28.993,
-      "step": 1250
-    },
-    {
-      "epoch": 5.17878204730275,
-      "grad_norm": 1.1285374164581299,
-      "learning_rate": 4.820717131474104e-05,
-      "loss": 1.0088,
-      "step": 1300
-    },
-    {
-      "epoch": 5.17878204730275,
-      "eval_loss": 1.0875197649002075,
-      "eval_runtime": 14.7256,
-      "eval_samples_per_second": 57.587,
-      "eval_steps_per_second": 28.793,
-      "step": 1300
-    },
-    {
-      "epoch": 5.377428766528028,
-      "grad_norm": 0.9647625684738159,
-      "learning_rate": 4.6215139442231074e-05,
-      "loss": 1.0,
-      "step": 1350
-    },
-    {
-      "epoch": 5.377428766528028,
-      "eval_loss": 1.0778887271881104,
-      "eval_runtime": 14.6228,
-      "eval_samples_per_second": 57.992,
-      "eval_steps_per_second": 28.996,
-      "step": 1350
-    },
-    {
-      "epoch": 5.5760754857533055,
-      "grad_norm": 1.074511170387268,
-      "learning_rate": 4.4223107569721116e-05,
-      "loss": 0.9925,
-      "step": 1400
-    },
-    {
-      "epoch": 5.5760754857533055,
-      "eval_loss": 1.0578875541687012,
-      "eval_runtime": 14.6013,
-      "eval_samples_per_second": 58.077,
-      "eval_steps_per_second": 29.038,
-      "step": 1400
-    },
-    {
-      "epoch": 5.774722204978583,
-      "grad_norm": 0.999279797077179,
-      "learning_rate": 4.223107569721116e-05,
-      "loss": 0.9831,
-      "step": 1450
-    },
-    {
-      "epoch": 5.774722204978583,
-      "eval_loss": 1.068110466003418,
-      "eval_runtime": 14.6065,
-      "eval_samples_per_second": 58.056,
-      "eval_steps_per_second": 29.028,
-      "step": 1450
-    },
-    {
-      "epoch": 5.973368924203861,
-      "grad_norm": 1.027130126953125,
-      "learning_rate": 4.02390438247012e-05,
-      "loss": 0.9744,
-      "step": 1500
-    },
-    {
-      "epoch": 5.973368924203861,
-      "eval_loss": 1.0516407489776611,
-      "eval_runtime": 14.5827,
-      "eval_samples_per_second": 58.151,
-      "eval_steps_per_second": 29.076,
-      "step": 1500
     }
   ],
   "logging_steps": 50,
-  "max_steps": 2510,
+  "max_steps": 1280,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 500,
@@ -476,7 +56,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.404255336140636e+17,
+  "total_flos": 3.5663179337957376e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d60cd4fa81843b4806dea3364d37d3df9835095733d168d051c0b135e77b91aa
+oid sha256:497104d84493788ba6f9029b34e87263e741d66179363d27985c9e12854dd130
 size 5368