Training in progress, step 500, checkpoint

last-checkpoint/config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "tattabio/gLM2_150M",
+  "_name_or_path": "tattabio/gLM2_650M",
   "architectures": [
     "gLM2ForMaskedLM"
   ],
@@ -8,10 +8,10 @@
     "AutoModel": "modeling_glm2.gLM2Model",
     "AutoModelForMaskedLM": "modeling_glm2.gLM2ForMaskedLM"
   },
-  "depth": 30,
-  "dim": 640,
+  "depth": 33,
+  "dim": 1280,
   "ffn_dim_multiplier": null,
-  "heads": 10,
+  "heads": 20,
   "model_type": "gLM2",
   "norm_eps": 1e-05,
   "swiglu_multiple_of": 256,

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47c3ea48a3d6443a0fd08b53e26ae04984ed6d6381f3928afee8fb4a155e9eaf
-size 609855088
+oid sha256:38384749b3fa8fe08d6d05430325f30ab4a6dfb4936f267fce232c98c501dc69
+size 2682482800

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ff76df2be9ebe7397bfb66a0d8ba41a51dc5c9909150e0d126337d9da67e16a
-size 1219840122
+oid sha256:493eb904d50aba9fced0e8ab2ed8fc66ac410087d78582e9bad1afa3df987de3
+size 5365108834

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e7034a159dfbc62e35de23e4c6fd1b9f4a30b049e2fbd1857b3a8bade788440
+oid sha256:848fdf35f13e1fde847fbd191021c99c0675e5e723a1b65fde4649f2fc9250db
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc2ced66205bf6a6509c630ed58ad5e3a6e3843df72048f8b326021e9e36307f
+oid sha256:d9f9fc41c0627d630837221d5c7872d3197c08985ee35f058d5f5e36bfe0249b
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:753dcfeefdbe4107c04bfd9f26c44afd642bd34e5d3d0b685a1b7e09f489639e
+oid sha256:7ae1449d711371210b0f6284f921f3df183a3c5c6628d3fc2950f5c89910866d
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e87981ec44568bc91dcff3b2572db474da39ce05df9ab33e26b3f375f4eac1b
+oid sha256:c3cc14e80a0475fa4dead8d6a3c6f0af9c5a92c40ad285584d68830834b3a6ea
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6956522915b3d6cfb4a36d77fb3f2d3b9a7b7a09c560fa745829757b1d0d295
+oid sha256:478ca537cf75a11344e25e46d3c46fdcf2db572bdb8cfff6f1ed3781e47a9787
 size 1064

last-checkpoint/trainer_state.json

@@ -1,152 +1,82 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.07672094673648273,
+  "epoch": 0.03836047336824137,
   "eval_steps": 500,
-  "global_step": 1000,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0038360473368241363,
-      "grad_norm": 60.51473617553711,
+      "grad_norm": 42.40585708618164,
       "learning_rate": 9.961638790854688e-05,
-      "loss": 82.5331,
+      "loss": 76.6444,
       "step": 50
     },
     {
       "epoch": 0.0076720946736482725,
-      "grad_norm": 41.30891418457031,
+      "grad_norm": 47.012596130371094,
       "learning_rate": 9.923277581709376e-05,
-      "loss": 79.5007,
+      "loss": 73.8558,
       "step": 100
     },
     {
       "epoch": 0.011508142010472408,
-      "grad_norm": 47.033897399902344,
+      "grad_norm": 37.02272033691406,
       "learning_rate": 9.884916372564063e-05,
-      "loss": 78.5655,
+      "loss": 72.8816,
       "step": 150
     },
     {
       "epoch": 0.015344189347296545,
-      "grad_norm": 47.13461685180664,
+      "grad_norm": 47.1681022644043,
       "learning_rate": 9.846555163418752e-05,
-      "loss": 77.9608,
+      "loss": 72.3309,
       "step": 200
     },
     {
       "epoch": 0.019180236684120684,
-      "grad_norm": 50.00185012817383,
+      "grad_norm": 61.863441467285156,
       "learning_rate": 9.80819395427344e-05,
-      "loss": 77.3352,
+      "loss": 71.639,
       "step": 250
     },
     {
       "epoch": 0.023016284020944817,
-      "grad_norm": 47.55207443237305,
+      "grad_norm": 65.1348648071289,
       "learning_rate": 9.769832745128127e-05,
-      "loss": 76.8788,
+      "loss": 71.186,
       "step": 300
     },
     {
       "epoch": 0.026852331357768953,
-      "grad_norm": 32.375267028808594,
+      "grad_norm": 54.1230583190918,
       "learning_rate": 9.731471535982815e-05,
-      "loss": 76.3247,
+      "loss": 70.5527,
       "step": 350
     },
     {
       "epoch": 0.03068837869459309,
-      "grad_norm": 26.361143112182617,
+      "grad_norm": 24.713777542114258,
       "learning_rate": 9.693110326837502e-05,
-      "loss": 76.1166,
+      "loss": 70.4304,
       "step": 400
     },
     {
       "epoch": 0.03452442603141723,
-      "grad_norm": 30.172746658325195,
+      "grad_norm": 33.9341926574707,
       "learning_rate": 9.65474911769219e-05,
-      "loss": 75.7178,
+      "loss": 69.9943,
       "step": 450
     },
     {
       "epoch": 0.03836047336824137,
-      "grad_norm": 48.87131118774414,
+      "grad_norm": 45.115230560302734,
       "learning_rate": 9.616387908546877e-05,
-      "loss": 75.2954,
+      "loss": 69.615,
       "step": 500
-    },
-    {
-      "epoch": 0.0421965207050655,
-      "grad_norm": 24.453649520874023,
-      "learning_rate": 9.578026699401565e-05,
-      "loss": 75.0015,
-      "step": 550
-    },
-    {
-      "epoch": 0.046032568041889634,
-      "grad_norm": 31.099306106567383,
-      "learning_rate": 9.539665490256254e-05,
-      "loss": 75.0084,
-      "step": 600
-    },
-    {
-      "epoch": 0.049868615378713774,
-      "grad_norm": 29.60244369506836,
-      "learning_rate": 9.501304281110941e-05,
-      "loss": 74.4046,
-      "step": 650
-    },
-    {
-      "epoch": 0.05370466271553791,
-      "grad_norm": 37.62842559814453,
-      "learning_rate": 9.462943071965629e-05,
-      "loss": 74.3024,
-      "step": 700
-    },
-    {
-      "epoch": 0.05754071005236205,
-      "grad_norm": 29.12676429748535,
-      "learning_rate": 9.424581862820317e-05,
-      "loss": 73.9017,
-      "step": 750
-    },
-    {
-      "epoch": 0.06137675738918618,
-      "grad_norm": 29.885427474975586,
-      "learning_rate": 9.386220653675004e-05,
-      "loss": 73.7941,
-      "step": 800
-    },
-    {
-      "epoch": 0.06521280472601032,
-      "grad_norm": 53.64329147338867,
-      "learning_rate": 9.347859444529692e-05,
-      "loss": 73.4727,
-      "step": 850
-    },
-    {
-      "epoch": 0.06904885206283445,
-      "grad_norm": 30.879840850830078,
-      "learning_rate": 9.309498235384379e-05,
-      "loss": 73.1386,
-      "step": 900
-    },
-    {
-      "epoch": 0.07288489939965859,
-      "grad_norm": 32.532352447509766,
-      "learning_rate": 9.271137026239067e-05,
-      "loss": 73.0574,
-      "step": 950
-    },
-    {
-      "epoch": 0.07672094673648273,
-      "grad_norm": 29.306671142578125,
-      "learning_rate": 9.232775817093756e-05,
-      "loss": 72.7158,
-      "step": 1000
     }
   ],
   "logging_steps": 50,
@@ -166,7 +96,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.895117780751483e+17,
+  "total_flos": 8.568157303923016e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1d0607b0ef32a7fb9356d7e9cea98c6b3f4ff9d058f42339926e1f0132b13a7
+oid sha256:eb6051eee29c31ca0092abe3bc9f1d55068e299690664ab4b9484f44a7a3cebf
 size 5240