Training in progress, step 2

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "tattabio/gLM2_650M",
+  "_name_or_path": "tattabio/gLM2_150M",
   "architectures": [
     "gLM2ForMaskedLM"
   ],
@@ -8,14 +8,14 @@
     "AutoModel": "modeling_glm2.gLM2Model",
     "AutoModelForMaskedLM": "modeling_glm2.gLM2ForMaskedLM"
   },
-  "depth": 33,
-  "dim": 1280,
+  "depth": 30,
+  "dim": 640,
   "ffn_dim_multiplier": null,
-  "heads": 20,
+  "heads": 10,
   "model_type": "gLM2",
   "norm_eps": 1e-05,
   "swiglu_multiple_of": 256,
   "torch_dtype": "float32",
-  "transformers_version": "4.45.2",
+  "transformers_version": "4.46.0",
   "vocab_size": 37
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9eae45eb43651c4ce612c5b264270a3ccdfbc48e1be2784320e0059c614c3cab
-size 2682482800
+oid sha256:516558ed7782de66fc542438abb1c93e159afd70a2aeb6571ce83cca423452b0
+size 609855088

runs/Oct22_14-42-35_torch-flash-large-0-5/events.out.tfevents.1729608158.torch-flash-large-0-5.15432.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:810807ab17e17d607934b36f382c07db6fdce6404d6609316f2cc595fe0b50b5
-size 30034
+oid sha256:795897e579db5094733b5e6a833553202120ad6e7282eb13a24e0a23837fd9dd
+size 31933

runs/Oct28_21-42-34_torch-flash-large-0-6/events.out.tfevents.1730151755.torch-flash-large-0-6.676.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12933faa94443b29d2eb1ec478a5f8fb657eeaa213446b52f73ac3d37515ce58
+size 5199

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65de12ccf65227e16c5ac7d6f4de8c23b93867370e90dd502a95ed85503923fb
+oid sha256:064b240ea07b11fb2a55256aa70c4f515e16a1e7de5972e80b77b98e19219a68
 size 5240