Training in progress, step 100, checkpoint

last-checkpoint/config.json

@@ -3,12 +3,12 @@
   "architectures": [
     "EsmForMaskedLM"
   ],
-  "attention_probs_dropout_prob": 0.01,
+  "attention_probs_dropout_prob": 0.1,
   "classifier_dropout": null,
   "emb_layer_norm_before": false,
   "esmfold_config": null,
   "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.01,
+  "hidden_dropout_prob": 0.1,
   "hidden_size": 1280,
   "initializer_range": 0.02,
   "intermediate_size": 5120,

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48475f083a5c2dd20122d335f8f1642d2b1704a4a54b55ecc266741d8af4decc
+oid sha256:a054bde49e1eaba391534cd5d6084e672a214d2bcdb35339a02e7f60b250444a
 size 2611614300

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd01a2371483ae6f6e5acd61fd57cc71db67bf7044ba7e1b6b06f33fe8ab5258
+oid sha256:88cdb9d1903a098ab709a4a31d88f0bc508b9856da309433ddbe56564fc7e4c6
 size 5213028466

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5c2bb2c73c38e1a76a17a11eb153818930706a055ece3df18411968d5d527a9
-size 14942
+oid sha256:e5980b31e2575fd555b9b5d35d9fc68ab477ebc20a6cd235cb9c30ed6d640a78
+size 15006

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd7d798f6dc160785a8da7f98e23574f8db4fbcce75b7717e46f600053dad5a1
+oid sha256:0001f7c15dabc5570df0e50376618b295ab961ad32150f9a463dd24a0d337e21
 size 1064

last-checkpoint/trainer_state.json

@@ -1,233 +1,25 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.624842568962117,
+  "epoch": 0.024993702758484677,
   "eval_steps": 500,
-  "global_step": 2500,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.024993702758484677,
-      "grad_norm": 0.821007490158081,
-      "learning_rate": 0.000997500624843789,
-      "loss": 88799130419.2,
+      "grad_norm": 1.2550193071365356,
+      "learning_rate": 9.875031242189454e-05,
+      "loss": 89312785858.56,
       "step": 100
-    },
-    {
-      "epoch": 0.049987405516969355,
-      "grad_norm": 0.7833845615386963,
-      "learning_rate": 0.000995001249687578,
-      "loss": 15026012815.36,
-      "step": 200
-    },
-    {
-      "epoch": 0.07498110827545404,
-      "grad_norm": 0.7531446218490601,
-      "learning_rate": 0.0009925018745313673,
-      "loss": 34681771786.24,
-      "step": 300
-    },
-    {
-      "epoch": 0.09997481103393871,
-      "grad_norm": 0.7915667295455933,
-      "learning_rate": 0.0009900024993751563,
-      "loss": 4205931724.8,
-      "step": 400
-    },
-    {
-      "epoch": 0.1249685137924234,
-      "grad_norm": 0.7665759921073914,
-      "learning_rate": 0.0009875031242189453,
-      "loss": 7206768476.16,
-      "step": 500
-    },
-    {
-      "epoch": 0.1249685137924234,
-      "eval_loss": NaN,
-      "eval_runtime": 451.579,
-      "eval_samples_per_second": 34.373,
-      "eval_steps_per_second": 34.373,
-      "step": 500
-    },
-    {
-      "epoch": 0.14996221655090808,
-      "grad_norm": 0.918053388595581,
-      "learning_rate": 0.0009850037490627343,
-      "loss": 96702698618.88,
-      "step": 600
-    },
-    {
-      "epoch": 0.17495591930939275,
-      "grad_norm": 0.7509938478469849,
-      "learning_rate": 0.0009825043739065233,
-      "loss": 2674936709.12,
-      "step": 700
-    },
-    {
-      "epoch": 0.19994962206787742,
-      "grad_norm": 0.9247342944145203,
-      "learning_rate": 0.0009800049987503125,
-      "loss": 24605596057.6,
-      "step": 800
-    },
-    {
-      "epoch": 0.22494332482636212,
-      "grad_norm": 0.7432388663291931,
-      "learning_rate": 0.0009775056235941015,
-      "loss": 382103401267.2,
-      "step": 900
-    },
-    {
-      "epoch": 0.2499370275848468,
-      "grad_norm": 0.6949240565299988,
-      "learning_rate": 0.0009750062484378905,
-      "loss": 32236791398.4,
-      "step": 1000
-    },
-    {
-      "epoch": 0.2499370275848468,
-      "eval_loss": NaN,
-      "eval_runtime": 424.2575,
-      "eval_samples_per_second": 36.586,
-      "eval_steps_per_second": 36.586,
-      "step": 1000
-    },
-    {
-      "epoch": 0.2749307303433315,
-      "grad_norm": 0.6821365356445312,
-      "learning_rate": 0.0009725068732816797,
-      "loss": 26628705484.8,
-      "step": 1100
-    },
-    {
-      "epoch": 0.29992443310181616,
-      "grad_norm": 0.6461954116821289,
-      "learning_rate": 0.0009700074981254687,
-      "loss": 27969317765.12,
-      "step": 1200
-    },
-    {
-      "epoch": 0.3249181358603008,
-      "grad_norm": 0.7664767503738403,
-      "learning_rate": 0.0009675081229692577,
-      "loss": 64947680706.56,
-      "step": 1300
-    },
-    {
-      "epoch": 0.3499118386187855,
-      "grad_norm": 0.7155385613441467,
-      "learning_rate": 0.0009650087478130467,
-      "loss": 21440355368.96,
-      "step": 1400
-    },
-    {
-      "epoch": 0.37490554137727017,
-      "grad_norm": 0.6681867837905884,
-      "learning_rate": 0.0009625093726568358,
-      "loss": 25921771274.24,
-      "step": 1500
-    },
-    {
-      "epoch": 0.37490554137727017,
-      "eval_loss": NaN,
-      "eval_runtime": 423.4797,
-      "eval_samples_per_second": 36.653,
-      "eval_steps_per_second": 36.653,
-      "step": 1500
-    },
-    {
-      "epoch": 0.39989924413575484,
-      "grad_norm": 0.6754324436187744,
-      "learning_rate": 0.0009600099975006249,
-      "loss": 673877977989.12,
-      "step": 1600
-    },
-    {
-      "epoch": 0.42489294689423956,
-      "grad_norm": 0.5371769070625305,
-      "learning_rate": 0.0009575106223444139,
-      "loss": 445900065341.44,
-      "step": 1700
-    },
-    {
-      "epoch": 0.44988664965272424,
-      "grad_norm": 0.750677227973938,
-      "learning_rate": 0.000955011247188203,
-      "loss": 35523568599.04,
-      "step": 1800
-    },
-    {
-      "epoch": 0.4748803524112089,
-      "grad_norm": 0.7535462975502014,
-      "learning_rate": 0.000952511872031992,
-      "loss": 46281166684.16,
-      "step": 1900
-    },
-    {
-      "epoch": 0.4998740551696936,
-      "grad_norm": 0.7034441828727722,
-      "learning_rate": 0.000950012496875781,
-      "loss": 342942380195.84,
-      "step": 2000
-    },
-    {
-      "epoch": 0.4998740551696936,
-      "eval_loss": NaN,
-      "eval_runtime": 423.9254,
-      "eval_samples_per_second": 36.615,
-      "eval_steps_per_second": 36.615,
-      "step": 2000
-    },
-    {
-      "epoch": 0.5248677579281783,
-      "grad_norm": 0.7051777839660645,
-      "learning_rate": 0.0009475131217195702,
-      "loss": 3791894609.92,
-      "step": 2100
-    },
-    {
-      "epoch": 0.549861460686663,
-      "grad_norm": 0.5590131878852844,
-      "learning_rate": 0.0009450137465633592,
-      "loss": 41969762959.36,
-      "step": 2200
-    },
-    {
-      "epoch": 0.5748551634451476,
-      "grad_norm": 0.6362671256065369,
-      "learning_rate": 0.0009425143714071483,
-      "loss": 43926135767.04,
-      "step": 2300
-    },
-    {
-      "epoch": 0.5998488662036323,
-      "grad_norm": 0.6453591585159302,
-      "learning_rate": 0.0009400149962509373,
-      "loss": 972726877552.64,
-      "step": 2400
-    },
-    {
-      "epoch": 0.624842568962117,
-      "grad_norm": 0.6738520264625549,
-      "learning_rate": 0.0009375156210947263,
-      "loss": 1207760825876.48,
-      "step": 2500
-    },
-    {
-      "epoch": 0.624842568962117,
-      "eval_loss": NaN,
-      "eval_runtime": 423.912,
-      "eval_samples_per_second": 36.616,
-      "eval_steps_per_second": 36.616,
-      "step": 2500
     }
   ],
   "logging_steps": 100,
-  "max_steps": 40010,
+  "max_steps": 8002,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 2,
   "save_steps": 100,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -241,7 +33,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.862199331815424e+17,
+  "total_flos": 1.9448797327261696e+16,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c699965ccc424515c6327ad161112409d54814998bcea28fb2549ae8ad3d0f20
+oid sha256:5e112b2855101d9f6ef3ac1291e6f721e5abc5c131c8763f4579e684682a8ab7
 size 5176