Upload 12 files

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "/users7/hcwang/2023/medchat/model/chatglm/original",
+  "architectures": [
+    "ChatGLMForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_chatglm.ChatGLMConfig",
+    "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration",
+    "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration"
+  },
+  "bos_token_id": 150004,
+  "eos_token_id": 150005,
+  "hidden_size": 4096,
+  "inner_hidden_size": 16384,
+  "layernorm_epsilon": 1e-05,
+  "max_sequence_length": 2048,
+  "model_type": "chatglm",
+  "num_attention_heads": 32,
+  "num_layers": 28,
+  "pad_token_id": 0,
+  "position_encoding_2d": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.27.1",
+  "use_cache": false,
+  "vocab_size": 150528
+}

configuration_chatglm.py

@@ -0,0 +1,92 @@
+""" ChatGLM model configuration """
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class ChatGLMConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`~ChatGLMModel`].
+    It is used to instantiate an ChatGLM model according to the specified arguments, defining the model
+    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+    the ChatGLM-6B [THUDM/ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) architecture.
+
+    Configuration objects inherit from  [`PretrainedConfig`] and can be used
+    to control the model outputs. Read the documentation from  [`PretrainedConfig`]
+    for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 150528):
+            Vocabulary size of the ChatGLM-6B model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`~ChatGLMModel`] or
+            [`~TFChatGLMModel`].
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the encoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 28):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        inner_hidden_size (`int`, *optional*, defaults to 16384):
+            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        max_sequence_length (`int`, *optional*, defaults to 512):
+            The maximum sequence length that this model might ever be used with.
+            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+        layernorm_epsilon (`float`, *optional*, defaults to 1e-5):
+            The epsilon used by the layer normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether the model should return the last key/values attentions (not used by all models).
+        Example:
+
+    ```python
+    >>> from configuration_chatglm import ChatGLMConfig
+    >>> from modeling_chatglm import ChatGLMModel
+
+    >>> # Initializing a ChatGLM-6B THUDM/ChatGLM-6B style configuration
+    >>> configuration = ChatGLMConfig()
+
+    >>> # Initializing a model from the THUDM/ChatGLM-6B style configuration
+    >>> model = ChatGLMModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```
+"""
+    model_type = "chatglm"
+
+    def __init__(
+            self,
+            vocab_size=150528,
+            hidden_size=4096,
+            num_layers=28,
+            num_attention_heads=32,
+            layernorm_epsilon=1e-5,
+            use_cache=False,
+            bos_token_id=150004,
+            eos_token_id=150005,
+            pad_token_id=0,
+            max_sequence_length=2048,
+            inner_hidden_size=16384,
+            position_encoding_2d=True,
+            **kwargs
+    ):
+        self.num_layers = num_layers
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_attention_heads = num_attention_heads
+        self.max_sequence_length = max_sequence_length
+        self.layernorm_epsilon = layernorm_epsilon
+        self.inner_hidden_size = inner_hidden_size
+        self.use_cache = use_cache
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.pad_token_id = pad_token_id
+        self.position_encoding_2d = position_encoding_2d
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs
+        )

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 150004,
+  "eos_token_id": 150005,
+  "pad_token_id": 0,
+  "transformers_version": "4.27.1"
+}

ice_text.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99871e0c85db81ad7af1028854fd091cd5778c8414ae9d94bbbc10d02c831c21
+size 2699926

pytorch_model.bin.index.json

@@ -0,0 +1,375 @@
+{
+  "metadata": {
+    "total_size": 13743539968
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.final_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.final_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.0.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.22.attention.dense.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.22.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.attention.dense.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.attention.dense.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.attention.dense.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.attention.dense.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.attention.dense.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.attention.query_key_value.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.post_attention_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.layers.3.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.attention.dense.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.attention.query_key_value.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.post_attention_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.word_embeddings.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6780eef001b060016bccd68cafdc28de890a53f520d257594e8ecef19f8fba49
+size 14639

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c78647fa1a01c6858012789e28ec1886bee43ca1a6383a9f454e3d8ee274221
+size 627

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "<sop>",
+  "eos_token": "</s>",
+  "mask_token": "[MASK]",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenization_chatglm.py

@@ -0,0 +1,346 @@
+"""Tokenization classes for ChatGLM."""
+import sys
+import unicodedata
+from typing import List, Optional, Union
+from functools import lru_cache
+import os
+import collections
+import re
+
+from transformers.tokenization_utils import PreTrainedTokenizer
+from icetk.text_tokenizer import TextTokenizer
+from icetk.utils import auto_create
+import icetk.sentencepiece_model_pb2 as sp_model
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "THUDM/chatglm-6b": 2048,
+}
+
+
+class SPTokenizer:
+    def __init__(
+        self,
+        vocab_file,
+        max_blank_length=80,
+        byte_fallback=True,
+    ):
+        assert vocab_file is not None
+        self.vocab_file = vocab_file
+        self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "<unused_0>", "<sop>", "<eop>", "<ENC>", "<dBLOCK>"]
+        self.max_blank_length = max_blank_length
+        self.byte_fallback = byte_fallback
+        self.text_tokenizer = self._build_text_tokenizer(encode_special_tokens=False)
+        self.special_text_tokenizer = self._build_text_tokenizer(encode_special_tokens=True)
+
+    @staticmethod
+    def _configure_tokenizer(
+        text_tokenizer: TextTokenizer,
+        special_tokens: List[str],
+        max_blank_length: int,
+        byte_fallback: bool,
+        encode_special_tokens=False,
+    ):
+        # special token
+        special_token_type = 4 if encode_special_tokens else 3  # 3 - CONTROL, 4 - USER_DEFINE
+        for token in special_tokens:
+            text_tokenizer.proto.pieces.append(
+                sp_model.ModelProto.SentencePiece(piece=token, score=0.0, type=special_token_type)
+            )
+        # whitespaces
+        for token in [SPTokenizer.get_tab_token()] + [
+            SPTokenizer.get_blank_token(i) for i in range(2, max_blank_length + 1)
+        ]:
+            text_tokenizer.proto.pieces.append(sp_model.ModelProto.SentencePiece(piece=token, score=0.0, type=4))
+        # byte fallback
+        if byte_fallback:
+            text_tokenizer.proto.trainer_spec.byte_fallback = True
+            for i in range(256):
+                text_tokenizer.proto.pieces.append(
+                    sp_model.ModelProto.SentencePiece(piece="<0x{:02X}>".format(i), score=0.0, type=6)
+                )
+        text_tokenizer.refresh()
+
+    def _build_text_tokenizer(self, encode_special_tokens=False):
+        tokenizer = TextTokenizer(self.vocab_file)
+        self._configure_tokenizer(
+            tokenizer, self.special_tokens, self.max_blank_length, self.byte_fallback, encode_special_tokens
+        )
+        return tokenizer
+
+    def _get_text_tokenizer(self, encode_special_tokens=False):
+        if encode_special_tokens:
+            return self.special_text_tokenizer
+        else:
+            return self.text_tokenizer
+
+    @staticmethod
+    def get_blank_token(length: int):
+        assert length >= 2
+        return f"<|blank_{length}|>"
+
+    @staticmethod
+    def get_tab_token():
+        return f"<|tab|>"
+
+    @property
+    def num_image_tokens(self):
+        return 20000
+
+    @property
+    def num_text_tokens(self):
+        return self.text_tokenizer.num_tokens
+
+    @property
+    def num_tokens(self):
+        return self.num_image_tokens + self.num_text_tokens
+
+    @staticmethod
+    def _encode_whitespaces(text: str, max_len: int = 80):
+        text = text.replace("\t", SPTokenizer.get_tab_token())
+        for i in range(max_len, 1, -1):
+            text = text.replace(" " * i, SPTokenizer.get_blank_token(i))
+        return text
+
+    def _preprocess(self, text: str, linebreak=True, whitespaces=True):
+        if linebreak:
+            text = text.replace("\n", "<n>")
+        if whitespaces:
+            text = self._encode_whitespaces(text, max_len=self.max_blank_length)
+        return text
+
+    def encode(
+        self, text: str, linebreak=True, whitespaces=True, special_tokens=False, add_dummy_prefix=True
+    ) -> List[int]:
+        """
+        @param text: Text to encode.
+        @param linebreak: Whether to encode newline (\n) in text.
+        @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
+        @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
+        @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
+        """
+        text = self._preprocess(text, linebreak, whitespaces)
+        if not add_dummy_prefix:
+            text = "<n>" + text
+        tmp = self._get_text_tokenizer(encode_special_tokens=special_tokens).encode(text)
+        tokens = [x + self.num_image_tokens for x in tmp]
+        return tokens if add_dummy_prefix else tokens[2:]
+
+    def decode(self, text_ids: List[int], special_tokens=False) -> str:
+        ids = [int(_id) - self.num_image_tokens for _id in text_ids]
+        ids = [_id for _id in ids if _id >= 0]
+        text = self._get_text_tokenizer(encode_special_tokens=special_tokens).decode(ids)
+        text = text.replace("<n>", "\n")
+        text = text.replace(SPTokenizer.get_tab_token(), "\t")
+        for i in range(2, self.max_blank_length + 1):
+            text = text.replace(self.get_blank_token(i), " " * i)
+        return text
+
+    def tokenize(
+        self, text: str, linebreak=True, whitespaces=True, special_tokens=False, add_dummy_prefix=True
+    ) -> List[str]:
+        """
+        @param text: Text to encode.
+        @param linebreak: Whether to encode newline (\n) in text.
+        @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
+        @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
+        @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
+        """
+        text = self._preprocess(text, linebreak, whitespaces)
+        if not add_dummy_prefix:
+            text = "<n>" + text
+        tokens = self._get_text_tokenizer(encode_special_tokens=special_tokens).tokenize(text)
+        return tokens if add_dummy_prefix else tokens[2:]
+
+    def __getitem__(self, x: Union[int, str]):
+        if isinstance(x, int):
+            if x < self.num_image_tokens:
+                return "<image_{}>".format(x)
+            else:
+                return self.text_tokenizer.convert_id_to_token(x - self.num_image_tokens)
+        elif isinstance(x, str):
+            if x.startswith("<image_") and x.endswith(">") and x[7:-1].isdigit():
+                return int(x[7:-1])
+            else:
+                return self.text_tokenizer.convert_token_to_id(x) + self.num_image_tokens
+        else:
+            raise ValueError("The key should be str or int.")
+
+
+class ChatGLMTokenizer(PreTrainedTokenizer):
+    """
+    Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = {"vocab_file": "ice_text.model"}
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    model_input_names = ["input_ids"]
+
+    def __init__(
+            self,
+            vocab_file,
+            do_lower_case=False,
+            remove_space=False,
+            bos_token='sop',
+            eos_token='eos',
+            eop_token='eop',
+            mask_token='[MASK]',
+            gmask_token='[gMASK]',
+            padding_side="left",
+            **kwargs
+    ) -> None:
+        super().__init__(
+            do_lower_case=do_lower_case,
+            remove_space=remove_space,
+            padding_side=padding_side,
+            **kwargs
+        )
+
+        self.do_lower_case = do_lower_case
+        self.remove_space = remove_space
+        self.vocab_file = vocab_file
+
+        self.bos_token = bos_token
+        self.eos_token = eos_token
+        self.eop_token = eop_token
+        self.mask_token = mask_token
+        self.gMASK_token = gmask_token
+
+        self.sp_tokenizer = SPTokenizer(vocab_file)
+
+        """ Initialisation """
+
+    @property
+    def eop_token_id(self) -> Optional[int]:
+        """
+        `Optional[int]`: Id of the end of sentence token in the vocabulary. Returns `None` if the token has not been
+        set.
+        """
+        if self.eop_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.eop_token)
+
+    @property
+    def vocab_size(self):
+        """ Returns vocab size """
+        return self.sp_tokenizer.num_tokens
+
+    def get_vocab(self):
+        """ Returns vocab as a dict """
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def preprocess_text(self, inputs):
+        if self.remove_space:
+            outputs = " ".join(inputs.strip().split())
+        else:
+            outputs = inputs
+
+        if self.do_lower_case:
+            outputs = outputs.lower()
+
+        return outputs
+
+    def _tokenize(self, text, **kwargs):
+        """ Returns a tokenized string. """
+        text = self.preprocess_text(text)
+
+        seq = self.sp_tokenizer.tokenize(text)
+
+        return seq
+
+    def decode(
+            self,
+            token_ids: Union[List[int], List[List[int]]],
+            skip_special_tokens: bool = False,
+            clean_up_tokenization_spaces: bool = True,
+            spaces_between_special_tokens: bool = True,
+            **kwargs
+    ) -> str:
+        if isinstance(token_ids[0], list):
+            tokens = []
+            for single_token_ids in token_ids:
+                if self.pad_token_id in single_token_ids:  # remove pad
+                    single_token_ids = list(filter((self.pad_token_id).__ne__, single_token_ids))
+                tokens.append(self.sp_tokenizer.decode(single_token_ids))
+            return (tokens)
+        else:
+            if self.pad_token_id in token_ids:  # remove pad
+                token_ids = list(filter((self.pad_token_id).__ne__, token_ids))
+            return self.sp_tokenizer.decode(token_ids)
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        return self.sp_tokenizer[token]
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.sp_tokenizer[index]
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+            filename_prefix (`str`, *optional*):
+                An optional prefix to add to the named of the saved files.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(
+                save_directory, self.vocab_files_names["vocab_file"]
+            )
+        else:
+            vocab_file = save_directory
+
+        with open(self.vocab_file, 'rb') as fin:
+            proto_str = fin.read()
+
+        with open(vocab_file, "wb") as writer:
+            writer.write(proto_str)
+
+        return (vocab_file,)
+
+    def build_inputs_with_special_tokens(
+            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. A BERT sequence has the following format:
+
+        - single sequence: `[CLS] X [SEP]`
+        - pair of sequences: `[CLS] A [SEP] B [SEP]`
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+        if token_ids_1 is not None:
+            token_ids_0 += token_ids_1
+        mask_ids = self.sp_tokenizer[self.mask_token]
+        gmask_ids = self.sp_tokenizer[self.gMASK_token]
+        if mask_ids not in token_ids_0 and gmask_ids not in token_ids_0:
+            token_ids_0 += [gmask_ids]
+
+        if token_ids_0[-1] != mask_ids and token_ids_0[-1] != gmask_ids:
+            token_ids_0 += [self.sp_tokenizer[self.eos_token]]
+
+        token_ids_0 += [self.sp_tokenizer[self.bos_token]]
+
+        return token_ids_0

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_chatglm.ChatGLMTokenizer",
+      null
+    ]
+  },
+  "do_lower_case": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "remove_space": false,
+  "special_tokens_map_file": null,
+  "tokenizer_class": "ChatGLMTokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,604 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "global_step": 953,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "learning_rate": 3.496503496503497e-06,
+      "loss": 5.6988,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 6.993006993006994e-06,
+      "loss": 5.6631,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.048951048951049e-05,
+      "loss": 5.7268,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.3986013986013988e-05,
+      "loss": 5.2938,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.7482517482517483e-05,
+      "loss": 5.0975,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 2.097902097902098e-05,
+      "loss": 4.8578,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 2.4475524475524478e-05,
+      "loss": 4.3282,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 2.7972027972027976e-05,
+      "loss": 4.0068,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 3.146853146853147e-05,
+      "loss": 3.7421,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 3.4965034965034965e-05,
+      "loss": 3.2654,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 3.846153846153846e-05,
+      "loss": 3.0331,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.195804195804196e-05,
+      "loss": 2.8466,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.545454545454546e-05,
+      "loss": 2.6892,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.8951048951048956e-05,
+      "loss": 2.7062,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.972762645914397e-05,
+      "loss": 2.6551,
+      "step": 150
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.933852140077821e-05,
+      "loss": 2.6632,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.894941634241245e-05,
+      "loss": 2.7111,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 4.856031128404669e-05,
+      "loss": 2.6711,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 4.817120622568094e-05,
+      "loss": 2.562,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 4.778210116731518e-05,
+      "loss": 2.543,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.739299610894942e-05,
+      "loss": 2.5899,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.700389105058366e-05,
+      "loss": 2.5576,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.66147859922179e-05,
+      "loss": 2.5863,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.622568093385214e-05,
+      "loss": 2.4993,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.583657587548638e-05,
+      "loss": 2.5329,
+      "step": 250
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.544747081712062e-05,
+      "loss": 2.5328,
+      "step": 260
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.505836575875487e-05,
+      "loss": 2.5523,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.466926070038911e-05,
+      "loss": 2.6414,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 4.428015564202335e-05,
+      "loss": 2.5356,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 4.389105058365759e-05,
+      "loss": 2.5604,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 4.3501945525291833e-05,
+      "loss": 2.5026,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 4.311284046692607e-05,
+      "loss": 2.5861,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.272373540856031e-05,
+      "loss": 2.6518,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.233463035019455e-05,
+      "loss": 2.4669,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.19455252918288e-05,
+      "loss": 2.4614,
+      "step": 350
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.155642023346304e-05,
+      "loss": 2.5185,
+      "step": 360
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.116731517509728e-05,
+      "loss": 2.5792,
+      "step": 370
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.077821011673152e-05,
+      "loss": 2.4151,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.0389105058365764e-05,
+      "loss": 2.5522,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4e-05,
+      "loss": 2.5522,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.961089494163424e-05,
+      "loss": 2.4389,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 3.922178988326848e-05,
+      "loss": 2.3915,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 3.883268482490273e-05,
+      "loss": 2.4208,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.844357976653697e-05,
+      "loss": 2.4861,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.805447470817121e-05,
+      "loss": 2.4987,
+      "step": 450
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.766536964980545e-05,
+      "loss": 2.5472,
+      "step": 460
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.7276264591439694e-05,
+      "loss": 2.5182,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.5792096979068868,
+      "eval_loss": 2.443359375,
+      "eval_runtime": 25.0491,
+      "eval_samples_per_second": 76.09,
+      "eval_steps_per_second": 9.541,
+      "step": 476
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 3.6887159533073934e-05,
+      "loss": 2.3115,
+      "step": 480
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 3.649805447470817e-05,
+      "loss": 2.3429,
+      "step": 490
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 3.610894941634241e-05,
+      "loss": 2.4361,
+      "step": 500
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 3.571984435797666e-05,
+      "loss": 2.4135,
+      "step": 510
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 3.53307392996109e-05,
+      "loss": 2.3602,
+      "step": 520
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 3.494163424124514e-05,
+      "loss": 2.4515,
+      "step": 530
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 3.455252918287938e-05,
+      "loss": 2.4394,
+      "step": 540
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 3.4163424124513624e-05,
+      "loss": 2.4532,
+      "step": 550
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 3.3774319066147864e-05,
+      "loss": 2.4665,
+      "step": 560
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 3.3385214007782103e-05,
+      "loss": 2.3228,
+      "step": 570
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 3.299610894941634e-05,
+      "loss": 2.3944,
+      "step": 580
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.260700389105058e-05,
+      "loss": 2.4102,
+      "step": 590
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 3.221789883268483e-05,
+      "loss": 2.4339,
+      "step": 600
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 3.182879377431907e-05,
+      "loss": 2.3893,
+      "step": 610
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 3.143968871595331e-05,
+      "loss": 2.3751,
+      "step": 620
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 3.105058365758755e-05,
+      "loss": 2.3861,
+      "step": 630
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 3.0661478599221794e-05,
+      "loss": 2.4282,
+      "step": 640
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 3.027237354085603e-05,
+      "loss": 2.3462,
+      "step": 650
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 2.9883268482490273e-05,
+      "loss": 2.3599,
+      "step": 660
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 2.9494163424124516e-05,
+      "loss": 2.4088,
+      "step": 670
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 2.910505836575876e-05,
+      "loss": 2.3874,
+      "step": 680
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 2.8715953307392995e-05,
+      "loss": 2.484,
+      "step": 690
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 2.832684824902724e-05,
+      "loss": 2.3859,
+      "step": 700
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.793774319066148e-05,
+      "loss": 2.3496,
+      "step": 710
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 2.7548638132295724e-05,
+      "loss": 2.5695,
+      "step": 720
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 2.715953307392996e-05,
+      "loss": 2.3911,
+      "step": 730
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 2.6770428015564204e-05,
+      "loss": 2.4357,
+      "step": 740
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 2.6381322957198447e-05,
+      "loss": 2.4545,
+      "step": 750
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 2.599221789883269e-05,
+      "loss": 2.3346,
+      "step": 760
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 2.5603112840466926e-05,
+      "loss": 2.3698,
+      "step": 770
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 2.521400778210117e-05,
+      "loss": 2.4074,
+      "step": 780
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 2.4824902723735412e-05,
+      "loss": 2.5796,
+      "step": 790
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 2.443579766536965e-05,
+      "loss": 2.4123,
+      "step": 800
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 2.4046692607003894e-05,
+      "loss": 2.3395,
+      "step": 810
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 2.3657587548638134e-05,
+      "loss": 2.3278,
+      "step": 820
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.3268482490272377e-05,
+      "loss": 2.4002,
+      "step": 830
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 2.2879377431906616e-05,
+      "loss": 2.3189,
+      "step": 840
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 2.2490272373540856e-05,
+      "loss": 2.4051,
+      "step": 850
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 2.21011673151751e-05,
+      "loss": 2.3595,
+      "step": 860
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 2.171206225680934e-05,
+      "loss": 2.3412,
+      "step": 870
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 2.132295719844358e-05,
+      "loss": 2.3462,
+      "step": 880
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 2.093385214007782e-05,
+      "loss": 2.3828,
+      "step": 890
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 2.054474708171206e-05,
+      "loss": 2.3796,
+      "step": 900
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 2.0155642023346304e-05,
+      "loss": 2.2987,
+      "step": 910
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 1.9766536964980543e-05,
+      "loss": 2.3002,
+      "step": 920
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 1.9377431906614786e-05,
+      "loss": 2.4586,
+      "step": 930
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 1.8988326848249026e-05,
+      "loss": 2.4058,
+      "step": 940
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 1.859922178988327e-05,
+      "loss": 2.3934,
+      "step": 950
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.5807643452808307,
+      "eval_loss": 2.384765625,
+      "eval_runtime": 25.0512,
+      "eval_samples_per_second": 76.084,
+      "eval_steps_per_second": 9.54,
+      "step": 953
+    }
+  ],
+  "max_steps": 1428,
+  "num_train_epochs": 3,
+  "total_flos": 5.11391603907625e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:164c847d97799194aedc004b3e7b6a4da88a9408da16201d2799d8e761fd15e2
+size 3515