Training in progress, step 2000

configuration_glm2.py

@@ -0,0 +1,37 @@
+"""gLM2 model configuration"""
+
+from typing import Optional
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class gLM2Config(PretrainedConfig):
+    model_type = "gLM2"
+
+    def __init__(
+        self,
+        dim: int = 640,
+        depth: int = 30,
+        heads: int = 10,
+        vocab_size: int = 4160,
+        swiglu_multiple_of: int = 256,
+        ffn_dim_multiplier: Optional[float] = None,
+        norm_eps: float = 1e-5,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.dim = dim
+        self.depth = depth
+        self.heads = heads
+        self.vocab_size = vocab_size
+        self.swiglu_multiple_of = swiglu_multiple_of
+        self.ffn_dim_multiplier = ffn_dim_multiplier
+        self.norm_eps = norm_eps
+
+        self.auto_map = {
+            "AutoConfig": "configuration_glm2.gLM2Config",
+            "AutoModel": "modeling_glm2.gLM2Model",
+            "AutoModelForMaskedLM": "modeling_glm2.gLM2ForMaskedLM"
+        }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e047e722c8d2a4adc9591b035031716df14784019153e5c2953617fb6ea1cafb
+oid sha256:961e9c84f5993fddb7d9787c91bd26ad5b2cfdc5ab6a3da55a42712133ed4001
 size 2682482800

runs/Oct22_14-42-35_torch-flash-large-0-5/events.out.tfevents.1729608158.torch-flash-large-0-5.15432.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ba87df1a5691e2bcdf01eb7945e888258aecf93451209bc8b4d93e26cc24c36
-size 11044
+oid sha256:76d16de0b494f8ee644099fa039dc15371de6e7d438d95cdd9e2c28d163d0f61
+size 13154