Upload folder using huggingface_hub

Pretrain_2B/added_tokens.json

@@ -0,0 +1,17 @@
+{
+  "</box>": 92552,
+  "</box_3d>": 92558,
+  "</img>": 92545,
+  "</quad>": 92548,
+  "</ref>": 92550,
+  "<IMG_CONTEXT>": 92546,
+  "<bbox_2d>": 92553,
+  "<bbox_2d_in>": 92555,
+  "<bbox_3d>": 92554,
+  "<box>": 92551,
+  "<box_3d>": 92557,
+  "<img>": 92544,
+  "<quad>": 92547,
+  "<query>": 92556,
+  "<ref>": 92549
+}

Pretrain_2B/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 1.2714912057394097,
+    "train_runtime": 6445.9977,
+    "train_samples": 194308,
+    "train_samples_per_second": 30.144,
+    "train_steps_per_second": 0.118
+}

Pretrain_2B/config.json

@@ -0,0 +1,202 @@
+{
+  "_commit_hash": null,
+  "_name_or_path": "./weight/InternVL2-2B",
+  "architectures": [
+    "InternVLChatModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_internvl_chat.InternVLChatConfig",
+    "AutoModel": "modeling_internvl_chat.InternVLChatModel",
+    "AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"
+  },
+  "downsample_ratio": 0.5,
+  "dynamic_image_size": false,
+  "force_image_size": 448,
+  "hidden_size": 2048,
+  "llm_config": {
+    "_name_or_path": "internlm/internlm2-chat-1_8b",
+    "add_cross_attention": false,
+    "architectures": [
+      "InternLM2ForCausalLM"
+    ],
+    "attn_implementation": "flash_attention_2",
+    "auto_map": {
+      "AutoConfig": "configuration_internlm2.InternLM2Config",
+      "AutoModel": "modeling_internlm2.InternLM2ForCausalLM",
+      "AutoModelForCausalLM": "modeling_internlm2.InternLM2ForCausalLM"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bias": false,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 8192,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "min_length": 0,
+    "model_type": "internlm2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 8,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 2,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "factor": 2.0,
+      "type": "dynamic"
+    },
+    "rope_theta": 1000000,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.37.2",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_cache": false,
+    "vocab_size": 92559
+  },
+  "max_dynamic_patch": 6,
+  "min_dynamic_patch": 1,
+  "model_type": "internvl_chat",
+  "pad2square": false,
+  "ps_version": "v2",
+  "select_layer": -1,
+  "template": "internlm2-chat",
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": null,
+  "use_backbone_lora": 0,
+  "use_llm_lora": 0,
+  "use_thumbnail": true,
+  "vision_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": [
+      "InternVisionModel"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "drop_path_rate": 0.0,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 448,
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "intern_vit_6b",
+    "no_repeat_ngram_size": 0,
+    "norm_type": "layer_norm",
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qk_normalization": false,
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.37.2",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_flash_attn": true
+  }
+}

Pretrain_2B/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    92542,
+    92543
+  ],
+  "transformers_version": "4.37.2"
+}

Pretrain_2B/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cace91965caa7c385840b038987bd700fb1b1674232c81dab5a7e2fd1c36c107
+size 4411620192

Pretrain_2B/special_tokens_map.json

@@ -0,0 +1,89 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>",
+    "<img>",
+    "</img>",
+    "<IMG_CONTEXT>",
+    "<quad>",
+    "</quad>",
+    "<ref>",
+    "</ref>",
+    "<box>",
+    "</box>",
+    {
+      "content": "<bbox_2d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<bbox_3d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<bbox_2d_in>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<query>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<box_3d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</box_3d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Pretrain_2B/tokenization_internlm2.py

@@ -0,0 +1,235 @@
+# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on transformers/src/transformers/models/llama/tokenization_llama.py
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes for InternLM."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+
+# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer
+class InternLM2Tokenizer(PreTrainedTokenizer):
+    """
+    Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ['input_ids', 'attention_mask']
+    _auto_class = 'AutoTokenizer'
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token='<unk>',
+        bos_token='<s>',
+        eos_token='</s>',
+        pad_token='</s>',
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=True,
+        add_eos_token=False,
+        decode_with_prefix_space=False,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self._no_prefix_space_tokens = None
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def no_prefix_space_tokens(self):
+        if self._no_prefix_space_tokens is None:
+            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+            self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith('▁')}
+        return self._no_prefix_space_tokens
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size"""
+        return self.sp_model.get_piece_size()
+
+    @property
+    def bos_token_id(self) -> Optional[int]:
+        return self.sp_model.bos_id()
+
+    @property
+    def eos_token_id(self) -> Optional[int]:
+        return self.sp_model.eos_id()
+
+    def get_vocab(self):
+        """Returns vocab as a dict"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def _maybe_add_prefix_space(self, tokens, decoded):
+        if tokens and tokens[0] not in self.no_prefix_space_tokens:
+            return ' ' + decoded
+        else:
+            return decoded
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ''
+        prev_is_special = False
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                if not prev_is_special:
+                    out_string += ' '
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        out_string = self.clean_up_tokenization(out_string)
+        out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string)
+        return out_string[1:]
+
+    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f'Vocabulary path ({save_directory}) should be a directory')
+            return
+        out_vocab_file = os.path.join(
+            save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']
+        )
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, 'wb') as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1
+
+        if self.add_eos_token:
+            output = output + [self.eos_token_id]
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        if token_ids_1 is None:
+            return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
+        use of token type ids, therefore a list of zeros is returned.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of zeros.
+        """
+        eos = [self.eos_token_id]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + eos) * [0]
+        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

Pretrain_2B/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f868398fc4e05ee1e8aeba95ddf18ddcc45b8bce55d5093bead5bbf80429b48b
+size 1477754

Pretrain_2B/tokenizer_config.json

@@ -0,0 +1,233 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92538": {
+      "content": "<|plugin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92539": {
+      "content": "<|interpreter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92540": {
+      "content": "<|action_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92541": {
+      "content": "<|action_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92542": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92543": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92544": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92545": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92546": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92547": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92548": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92549": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92550": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92551": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92552": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92553": {
+      "content": "<bbox_2d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92554": {
+      "content": "<bbox_3d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92555": {
+      "content": "<bbox_2d_in>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92556": {
+      "content": "<query>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92557": {
+      "content": "<box_3d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92558": {
+      "content": "</box_3d>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>",
+    "<img>",
+    "</img>",
+    "<IMG_CONTEXT>",
+    "<quad>",
+    "</quad>",
+    "<ref>",
+    "</ref>",
+    "<box>",
+    "</box>",
+    "<bbox_2d>",
+    "<bbox_3d>",
+    "<bbox_2d_in>",
+    "<query>",
+    "<box_3d>",
+    "</box_3d>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_internlm2.InternLM2Tokenizer",
+      null
+    ]
+  },
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "tokenizer_class": "InternLM2Tokenizer",
+  "unk_token": "<unk>"
+}

Pretrain_2B/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 1.2714912057394097,
+    "train_runtime": 6445.9977,
+    "train_samples": 194308,
+    "train_samples_per_second": 30.144,
+    "train_steps_per_second": 0.118
+}

Pretrain_2B/trainer_state.json

@@ -0,0 +1,4584 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9999176615891313,
+  "eval_steps": 500,
+  "global_step": 759,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.695652173913044e-07,
+      "loss": 1.8484,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.7391304347826088e-06,
+      "loss": 1.9125,
+      "step": 2
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6086956521739132e-06,
+      "loss": 1.8943,
+      "step": 3
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.4782608695652175e-06,
+      "loss": 1.9085,
+      "step": 4
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.347826086956522e-06,
+      "loss": 1.887,
+      "step": 5
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.2173913043478265e-06,
+      "loss": 1.8607,
+      "step": 6
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.086956521739132e-06,
+      "loss": 1.8876,
+      "step": 7
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.956521739130435e-06,
+      "loss": 1.8753,
+      "step": 8
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.82608695652174e-06,
+      "loss": 1.8336,
+      "step": 9
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.695652173913044e-06,
+      "loss": 1.8016,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.565217391304349e-06,
+      "loss": 1.7738,
+      "step": 11
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0434782608695653e-05,
+      "loss": 1.7794,
+      "step": 12
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1304347826086957e-05,
+      "loss": 1.8015,
+      "step": 13
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2173913043478263e-05,
+      "loss": 1.7103,
+      "step": 14
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.3043478260869566e-05,
+      "loss": 1.7215,
+      "step": 15
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.391304347826087e-05,
+      "loss": 1.6887,
+      "step": 16
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.4782608695652174e-05,
+      "loss": 1.6018,
+      "step": 17
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.565217391304348e-05,
+      "loss": 1.6769,
+      "step": 18
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6521739130434785e-05,
+      "loss": 1.6393,
+      "step": 19
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.739130434782609e-05,
+      "loss": 1.6076,
+      "step": 20
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.8260869565217393e-05,
+      "loss": 1.5988,
+      "step": 21
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.9130434782608697e-05,
+      "loss": 1.6057,
+      "step": 22
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 2e-05,
+      "loss": 1.6224,
+      "step": 23
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.999990890103106e-05,
+      "loss": 1.6007,
+      "step": 24
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.9999635605784042e-05,
+      "loss": 1.5796,
+      "step": 25
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.9999180119238327e-05,
+      "loss": 1.5841,
+      "step": 26
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9998542449692794e-05,
+      "loss": 1.5519,
+      "step": 27
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.999772260876564e-05,
+      "loss": 1.519,
+      "step": 28
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.99967206113942e-05,
+      "loss": 1.5497,
+      "step": 29
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9995536475834667e-05,
+      "loss": 1.4893,
+      "step": 30
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.999417022366174e-05,
+      "loss": 1.5149,
+      "step": 31
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9992621879768256e-05,
+      "loss": 1.4988,
+      "step": 32
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.999089147236472e-05,
+      "loss": 1.4931,
+      "step": 33
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.998897903297879e-05,
+      "loss": 1.4954,
+      "step": 34
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.998688459645473e-05,
+      "loss": 1.47,
+      "step": 35
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.9984608200952736e-05,
+      "loss": 1.4654,
+      "step": 36
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.9982149887948264e-05,
+      "loss": 1.4511,
+      "step": 37
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.997950970223127e-05,
+      "loss": 1.4411,
+      "step": 38
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.9976687691905394e-05,
+      "loss": 1.4413,
+      "step": 39
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.997368390838708e-05,
+      "loss": 1.4492,
+      "step": 40
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.997049840640465e-05,
+      "loss": 1.4415,
+      "step": 41
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.996713124399729e-05,
+      "loss": 1.433,
+      "step": 42
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.9963582482514003e-05,
+      "loss": 1.4392,
+      "step": 43
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.9959852186612492e-05,
+      "loss": 1.4697,
+      "step": 44
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.995594042425798e-05,
+      "loss": 1.4352,
+      "step": 45
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.995184726672197e-05,
+      "loss": 1.3998,
+      "step": 46
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.994757278858095e-05,
+      "loss": 1.4069,
+      "step": 47
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.994311706771503e-05,
+      "loss": 1.4385,
+      "step": 48
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.993848018530652e-05,
+      "loss": 1.4026,
+      "step": 49
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.993366222583847e-05,
+      "loss": 1.3766,
+      "step": 50
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.99286632770931e-05,
+      "loss": 1.4277,
+      "step": 51
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.992348343015023e-05,
+      "loss": 1.3874,
+      "step": 52
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.99181227793856e-05,
+      "loss": 1.3954,
+      "step": 53
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.991258142246917e-05,
+      "loss": 1.3814,
+      "step": 54
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.9906859460363307e-05,
+      "loss": 1.3968,
+      "step": 55
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.990095699732099e-05,
+      "loss": 1.4273,
+      "step": 56
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9894874140883877e-05,
+      "loss": 1.3965,
+      "step": 57
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9888611001880357e-05,
+      "loss": 1.3884,
+      "step": 58
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.988216769442353e-05,
+      "loss": 1.3648,
+      "step": 59
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.987554433590913e-05,
+      "loss": 1.3636,
+      "step": 60
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9868741047013382e-05,
+      "loss": 1.4075,
+      "step": 61
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9861757951690813e-05,
+      "loss": 1.3511,
+      "step": 62
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9854595177171968e-05,
+      "loss": 1.3854,
+      "step": 63
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9847252853961136e-05,
+      "loss": 1.3692,
+      "step": 64
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.983973111583392e-05,
+      "loss": 1.377,
+      "step": 65
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.983203009983484e-05,
+      "loss": 1.3646,
+      "step": 66
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9824149946274827e-05,
+      "loss": 1.3667,
+      "step": 67
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9816090798728648e-05,
+      "loss": 1.3736,
+      "step": 68
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9807852804032306e-05,
+      "loss": 1.3783,
+      "step": 69
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9799436112280374e-05,
+      "loss": 1.3666,
+      "step": 70
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.979084087682323e-05,
+      "loss": 1.3737,
+      "step": 71
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.97820672542643e-05,
+      "loss": 1.3566,
+      "step": 72
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9773115404457175e-05,
+      "loss": 1.3695,
+      "step": 73
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9763985490502714e-05,
+      "loss": 1.3926,
+      "step": 74
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9754677678746064e-05,
+      "loss": 1.3779,
+      "step": 75
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9745192138773633e-05,
+      "loss": 1.3873,
+      "step": 76
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9735529043410012e-05,
+      "loss": 1.3452,
+      "step": 77
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.97256885687148e-05,
+      "loss": 1.3324,
+      "step": 78
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9715670893979416e-05,
+      "loss": 1.343,
+      "step": 79
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.970547620172383e-05,
+      "loss": 1.3607,
+      "step": 80
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9695104677693234e-05,
+      "loss": 1.3275,
+      "step": 81
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9684556510854655e-05,
+      "loss": 1.3336,
+      "step": 82
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.967383189339352e-05,
+      "loss": 1.34,
+      "step": 83
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9662931020710138e-05,
+      "loss": 1.3204,
+      "step": 84
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9651854091416175e-05,
+      "loss": 1.3353,
+      "step": 85
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.964060130733099e-05,
+      "loss": 1.3148,
+      "step": 86
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9629172873477995e-05,
+      "loss": 1.3188,
+      "step": 87
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9617568998080893e-05,
+      "loss": 1.3446,
+      "step": 88
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9605789892559902e-05,
+      "loss": 1.3092,
+      "step": 89
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9593835771527893e-05,
+      "loss": 1.3817,
+      "step": 90
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9581706852786492e-05,
+      "loss": 1.3141,
+      "step": 91
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.956940335732209e-05,
+      "loss": 1.3394,
+      "step": 92
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9556925509301844e-05,
+      "loss": 1.3468,
+      "step": 93
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9544273536069573e-05,
+      "loss": 1.3313,
+      "step": 94
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.953144766814161e-05,
+      "loss": 1.3337,
+      "step": 95
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9518448139202632e-05,
+      "loss": 1.3262,
+      "step": 96
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9505275186101378e-05,
+      "loss": 1.3225,
+      "step": 97
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9491929048846328e-05,
+      "loss": 1.3459,
+      "step": 98
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.947840997060136e-05,
+      "loss": 1.3039,
+      "step": 99
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9464718197681284e-05,
+      "loss": 1.2979,
+      "step": 100
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9450853979547384e-05,
+      "loss": 1.2875,
+      "step": 101
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9436817568802854e-05,
+      "loss": 1.3223,
+      "step": 102
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9422609221188208e-05,
+      "loss": 1.3165,
+      "step": 103
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.94082291955766e-05,
+      "loss": 1.2915,
+      "step": 104
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9393677753969137e-05,
+      "loss": 1.3495,
+      "step": 105
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9378955161490086e-05,
+      "loss": 1.3032,
+      "step": 106
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9364061686382042e-05,
+      "loss": 1.3499,
+      "step": 107
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9348997600001052e-05,
+      "loss": 1.3293,
+      "step": 108
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9333763176811663e-05,
+      "loss": 1.3426,
+      "step": 109
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9318358694381926e-05,
+      "loss": 1.3425,
+      "step": 110
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.9302784433378333e-05,
+      "loss": 1.2916,
+      "step": 111
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.928704067756071e-05,
+      "loss": 1.3192,
+      "step": 112
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.9271127713777033e-05,
+      "loss": 1.3236,
+      "step": 113
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.925504583195823e-05,
+      "loss": 1.3199,
+      "step": 114
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.9238795325112867e-05,
+      "loss": 1.2867,
+      "step": 115
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.922237648932183e-05,
+      "loss": 1.3383,
+      "step": 116
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.9205789623732923e-05,
+      "loss": 1.3369,
+      "step": 117
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.918903503055541e-05,
+      "loss": 1.3649,
+      "step": 118
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.917211301505453e-05,
+      "loss": 1.3467,
+      "step": 119
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9155023885545914e-05,
+      "loss": 1.3158,
+      "step": 120
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.913776795338998e-05,
+      "loss": 1.3026,
+      "step": 121
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9120345532986243e-05,
+      "loss": 1.337,
+      "step": 122
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9102756941767625e-05,
+      "loss": 1.293,
+      "step": 123
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.908500250019462e-05,
+      "loss": 1.3328,
+      "step": 124
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9067082531749496e-05,
+      "loss": 1.2999,
+      "step": 125
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.9048997362930384e-05,
+      "loss": 1.2873,
+      "step": 126
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.903074732324533e-05,
+      "loss": 1.3122,
+      "step": 127
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.901233274520629e-05,
+      "loss": 1.2823,
+      "step": 128
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.8993753964323086e-05,
+      "loss": 1.2737,
+      "step": 129
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.8975011319097264e-05,
+      "loss": 1.3172,
+      "step": 130
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.8956105151015966e-05,
+      "loss": 1.2967,
+      "step": 131
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.893703580454567e-05,
+      "loss": 1.2561,
+      "step": 132
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.891780362712594e-05,
+      "loss": 1.2895,
+      "step": 133
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8898408969163078e-05,
+      "loss": 1.3122,
+      "step": 134
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8878852184023754e-05,
+      "loss": 1.2937,
+      "step": 135
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8859133628028564e-05,
+      "loss": 1.3323,
+      "step": 136
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8839253660445523e-05,
+      "loss": 1.2676,
+      "step": 137
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.881921264348355e-05,
+      "loss": 1.3212,
+      "step": 138
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.879901094228584e-05,
+      "loss": 1.2788,
+      "step": 139
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8778648924923222e-05,
+      "loss": 1.2941,
+      "step": 140
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.875812696238745e-05,
+      "loss": 1.3259,
+      "step": 141
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.8737445428584456e-05,
+      "loss": 1.309,
+      "step": 142
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.8716604700327516e-05,
+      "loss": 1.303,
+      "step": 143
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.8695605157330398e-05,
+      "loss": 1.3137,
+      "step": 144
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.8674447182200457e-05,
+      "loss": 1.2694,
+      "step": 145
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.8653131160431622e-05,
+      "loss": 1.3115,
+      "step": 146
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.863165748039743e-05,
+      "loss": 1.3306,
+      "step": 147
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.861002653334389e-05,
+      "loss": 1.3016,
+      "step": 148
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.85882387133824e-05,
+      "loss": 1.2957,
+      "step": 149
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.8566294417482552e-05,
+      "loss": 1.2838,
+      "step": 150
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.8544194045464888e-05,
+      "loss": 1.2714,
+      "step": 151
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.8521937999993627e-05,
+      "loss": 1.2949,
+      "step": 152
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.849952668656933e-05,
+      "loss": 1.2796,
+      "step": 153
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.847696051352151e-05,
+      "loss": 1.2714,
+      "step": 154
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.845423989200118e-05,
+      "loss": 1.2807,
+      "step": 155
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8431365235973383e-05,
+      "loss": 1.2791,
+      "step": 156
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.840833696220963e-05,
+      "loss": 1.3286,
+      "step": 157
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8385155490280327e-05,
+      "loss": 1.2965,
+      "step": 158
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.836182124254711e-05,
+      "loss": 1.2916,
+      "step": 159
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.833833464415516e-05,
+      "loss": 1.2897,
+      "step": 160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8314696123025456e-05,
+      "loss": 1.2973,
+      "step": 161
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8290906109846974e-05,
+      "loss": 1.3001,
+      "step": 162
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8266965038068856e-05,
+      "loss": 1.2569,
+      "step": 163
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8242873343892494e-05,
+      "loss": 1.2652,
+      "step": 164
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8218631466263584e-05,
+      "loss": 1.2796,
+      "step": 165
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8194239846864133e-05,
+      "loss": 1.2463,
+      "step": 166
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.816969893010442e-05,
+      "loss": 1.2678,
+      "step": 167
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8145009163114894e-05,
+      "loss": 1.2939,
+      "step": 168
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.812017099573801e-05,
+      "loss": 1.3116,
+      "step": 169
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8095184880520058e-05,
+      "loss": 1.2601,
+      "step": 170
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.8070051272702905e-05,
+      "loss": 1.2895,
+      "step": 171
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.8044770630215706e-05,
+      "loss": 1.2686,
+      "step": 172
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.801934341366655e-05,
+      "loss": 1.3025,
+      "step": 173
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.7993770086334082e-05,
+      "loss": 1.2832,
+      "step": 174
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.7968051114159046e-05,
+      "loss": 1.3023,
+      "step": 175
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.794218696573582e-05,
+      "loss": 1.2325,
+      "step": 176
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.791617811230385e-05,
+      "loss": 1.267,
+      "step": 177
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.7890025027739084e-05,
+      "loss": 1.2639,
+      "step": 178
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.7863728188545326e-05,
+      "loss": 1.2909,
+      "step": 179
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.7837288073845566e-05,
+      "loss": 1.2805,
+      "step": 180
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.7810705165373245e-05,
+      "loss": 1.2607,
+      "step": 181
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.778397994746347e-05,
+      "loss": 1.2688,
+      "step": 182
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.77571129070442e-05,
+      "loss": 1.282,
+      "step": 183
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.773010453362737e-05,
+      "loss": 1.2594,
+      "step": 184
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.770295531929998e-05,
+      "loss": 1.2496,
+      "step": 185
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.767566575871511e-05,
+      "loss": 1.2712,
+      "step": 186
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7648236349082928e-05,
+      "loss": 1.2908,
+      "step": 187
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7620667590161626e-05,
+      "loss": 1.3051,
+      "step": 188
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.75929599842483e-05,
+      "loss": 1.2698,
+      "step": 189
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.756511403616982e-05,
+      "loss": 1.2813,
+      "step": 190
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7537130253273613e-05,
+      "loss": 1.2748,
+      "step": 191
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.750900914541844e-05,
+      "loss": 1.292,
+      "step": 192
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7480751224965083e-05,
+      "loss": 1.2547,
+      "step": 193
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7452357006767026e-05,
+      "loss": 1.2556,
+      "step": 194
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.742382700816107e-05,
+      "loss": 1.2535,
+      "step": 195
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7395161748957905e-05,
+      "loss": 1.264,
+      "step": 196
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7366361751432645e-05,
+      "loss": 1.2801,
+      "step": 197
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7337427540315305e-05,
+      "loss": 1.2835,
+      "step": 198
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.730835964278124e-05,
+      "loss": 1.2559,
+      "step": 199
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7279158588441558e-05,
+      "loss": 1.2278,
+      "step": 200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7249824909333445e-05,
+      "loss": 1.2496,
+      "step": 201
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.722035913991048e-05,
+      "loss": 1.2728,
+      "step": 202
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.719076181703291e-05,
+      "loss": 1.2566,
+      "step": 203
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.716103347995785e-05,
+      "loss": 1.2786,
+      "step": 204
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.713117467032948e-05,
+      "loss": 1.2873,
+      "step": 205
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.7101185932169147e-05,
+      "loss": 1.2745,
+      "step": 206
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.7071067811865477e-05,
+      "loss": 1.249,
+      "step": 207
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.7040820858164413e-05,
+      "loss": 1.2347,
+      "step": 208
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.7010445622159214e-05,
+      "loss": 1.2614,
+      "step": 209
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.6979942657280414e-05,
+      "loss": 1.2607,
+      "step": 210
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.694931251928575e-05,
+      "loss": 1.2215,
+      "step": 211
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.691855576625001e-05,
+      "loss": 1.2279,
+      "step": 212
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.68876729585549e-05,
+      "loss": 1.2378,
+      "step": 213
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.6856664658878797e-05,
+      "loss": 1.2437,
+      "step": 214
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.6825531432186545e-05,
+      "loss": 1.2671,
+      "step": 215
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.6794273845719096e-05,
+      "loss": 1.251,
+      "step": 216
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6762892468983237e-05,
+      "loss": 1.2588,
+      "step": 217
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.673138787374119e-05,
+      "loss": 1.2078,
+      "step": 218
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6699760634000166e-05,
+      "loss": 1.2534,
+      "step": 219
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6668011326001962e-05,
+      "loss": 1.2941,
+      "step": 220
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6636140528212427e-05,
+      "loss": 1.2867,
+      "step": 221
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6604148821310912e-05,
+      "loss": 1.2647,
+      "step": 222
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6572036788179728e-05,
+      "loss": 1.2414,
+      "step": 223
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6539805013893493e-05,
+      "loss": 1.2572,
+      "step": 224
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.650745408570849e-05,
+      "loss": 1.195,
+      "step": 225
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6474984593051965e-05,
+      "loss": 1.2229,
+      "step": 226
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6442397127511366e-05,
+      "loss": 1.2675,
+      "step": 227
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6409692282823604e-05,
+      "loss": 1.2349,
+      "step": 228
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.63768706548642e-05,
+      "loss": 1.2357,
+      "step": 229
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6343932841636455e-05,
+      "loss": 1.2307,
+      "step": 230
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.631087944326053e-05,
+      "loss": 1.2892,
+      "step": 231
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6277711061962525e-05,
+      "loss": 1.2696,
+      "step": 232
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6244428302063506e-05,
+      "loss": 1.2858,
+      "step": 233
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6211031769968503e-05,
+      "loss": 1.2654,
+      "step": 234
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6177522074155436e-05,
+      "loss": 1.274,
+      "step": 235
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6143899825164058e-05,
+      "loss": 1.255,
+      "step": 236
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6110165635584807e-05,
+      "loss": 1.2317,
+      "step": 237
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6076320120047667e-05,
+      "loss": 1.2737,
+      "step": 238
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6042363895210948e-05,
+      "loss": 1.2599,
+      "step": 239
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.6008297579750063e-05,
+      "loss": 1.2414,
+      "step": 240
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.597412179434626e-05,
+      "loss": 1.2389,
+      "step": 241
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5939837161675297e-05,
+      "loss": 1.2738,
+      "step": 242
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.590544430639611e-05,
+      "loss": 1.2611,
+      "step": 243
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5870943855139437e-05,
+      "loss": 1.2908,
+      "step": 244
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5836336436496377e-05,
+      "loss": 1.2377,
+      "step": 245
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5801622681006966e-05,
+      "loss": 1.2655,
+      "step": 246
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5766803221148676e-05,
+      "loss": 1.2419,
+      "step": 247
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5731878691324874e-05,
+      "loss": 1.2521,
+      "step": 248
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5696849727853297e-05,
+      "loss": 1.2574,
+      "step": 249
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5661716968954436e-05,
+      "loss": 1.2392,
+      "step": 250
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5626481054739916e-05,
+      "loss": 1.2319,
+      "step": 251
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5591142627200825e-05,
+      "loss": 1.2687,
+      "step": 252
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5555702330196024e-05,
+      "loss": 1.2643,
+      "step": 253
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.552016080944042e-05,
+      "loss": 1.2178,
+      "step": 254
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5484518712493188e-05,
+      "loss": 1.2774,
+      "step": 255
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.544877668874599e-05,
+      "loss": 1.2382,
+      "step": 256
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5412935389411124e-05,
+      "loss": 1.2719,
+      "step": 257
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5376995467509673e-05,
+      "loss": 1.2459,
+      "step": 258
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5340957577859605e-05,
+      "loss": 1.2503,
+      "step": 259
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.530482237706383e-05,
+      "loss": 1.2523,
+      "step": 260
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.526859052349827e-05,
+      "loss": 1.2432,
+      "step": 261
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5232262677299816e-05,
+      "loss": 1.2565,
+      "step": 262
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5195839500354337e-05,
+      "loss": 1.219,
+      "step": 263
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5159321656284602e-05,
+      "loss": 1.2263,
+      "step": 264
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5122709810438205e-05,
+      "loss": 1.2517,
+      "step": 265
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5086004629875426e-05,
+      "loss": 1.2761,
+      "step": 266
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5049206783357082e-05,
+      "loss": 1.2597,
+      "step": 267
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.501231694133235e-05,
+      "loss": 1.2765,
+      "step": 268
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.4975335775926547e-05,
+      "loss": 1.2937,
+      "step": 269
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4938263960928878e-05,
+      "loss": 1.2656,
+      "step": 270
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4901102171780175e-05,
+      "loss": 1.2233,
+      "step": 271
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4863851085560563e-05,
+      "loss": 1.2493,
+      "step": 272
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4826511380977155e-05,
+      "loss": 1.2665,
+      "step": 273
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.478908373835167e-05,
+      "loss": 1.2582,
+      "step": 274
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4751568839608036e-05,
+      "loss": 1.2779,
+      "step": 275
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4713967368259981e-05,
+      "loss": 1.2421,
+      "step": 276
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4676280009398544e-05,
+      "loss": 1.232,
+      "step": 277
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4638507449679642e-05,
+      "loss": 1.2347,
+      "step": 278
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4600650377311523e-05,
+      "loss": 1.2076,
+      "step": 279
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4562709482042237e-05,
+      "loss": 1.2774,
+      "step": 280
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4524685455147071e-05,
+      "loss": 1.221,
+      "step": 281
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.448657898941596e-05,
+      "loss": 1.2758,
+      "step": 282
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4448390779140844e-05,
+      "loss": 1.2019,
+      "step": 283
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4410121520103045e-05,
+      "loss": 1.2473,
+      "step": 284
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4371771909560566e-05,
+      "loss": 1.2239,
+      "step": 285
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4333342646235407e-05,
+      "loss": 1.2389,
+      "step": 286
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4294834430300822e-05,
+      "loss": 1.2811,
+      "step": 287
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.425624796336856e-05,
+      "loss": 1.2226,
+      "step": 288
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4217583948476094e-05,
+      "loss": 1.2374,
+      "step": 289
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4178843090073802e-05,
+      "loss": 1.2145,
+      "step": 290
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4140026094012136e-05,
+      "loss": 1.2324,
+      "step": 291
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4101133667528761e-05,
+      "loss": 1.2144,
+      "step": 292
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.4062166519235665e-05,
+      "loss": 1.2395,
+      "step": 293
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.4023125359106253e-05,
+      "loss": 1.2463,
+      "step": 294
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3984010898462417e-05,
+      "loss": 1.2726,
+      "step": 295
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3944823849961557e-05,
+      "loss": 1.2394,
+      "step": 296
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3905564927583625e-05,
+      "loss": 1.2473,
+      "step": 297
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3866234846618083e-05,
+      "loss": 1.2524,
+      "step": 298
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3826834323650899e-05,
+      "loss": 1.2286,
+      "step": 299
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3787364076551478e-05,
+      "loss": 1.2528,
+      "step": 300
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3747824824459577e-05,
+      "loss": 1.2363,
+      "step": 301
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3708217287772227e-05,
+      "loss": 1.2293,
+      "step": 302
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3668542188130567e-05,
+      "loss": 1.2363,
+      "step": 303
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3628800248406738e-05,
+      "loss": 1.2268,
+      "step": 304
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3588992192690683e-05,
+      "loss": 1.2158,
+      "step": 305
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3549118746276968e-05,
+      "loss": 1.2529,
+      "step": 306
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.350918063565157e-05,
+      "loss": 1.2637,
+      "step": 307
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3469178588478621e-05,
+      "loss": 1.25,
+      "step": 308
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3429113333587181e-05,
+      "loss": 1.2452,
+      "step": 309
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3388985600957922e-05,
+      "loss": 1.2186,
+      "step": 310
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3348796121709862e-05,
+      "loss": 1.2639,
+      "step": 311
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3308545628087029e-05,
+      "loss": 1.253,
+      "step": 312
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3268234853445113e-05,
+      "loss": 1.242,
+      "step": 313
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3227864532238113e-05,
+      "loss": 1.2593,
+      "step": 314
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.318743540000496e-05,
+      "loss": 1.2187,
+      "step": 315
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.3146948193356105e-05,
+      "loss": 1.2392,
+      "step": 316
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.3106403649960109e-05,
+      "loss": 1.2819,
+      "step": 317
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.3065802508530186e-05,
+      "loss": 1.2638,
+      "step": 318
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.302514550881076e-05,
+      "loss": 1.2472,
+      "step": 319
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.2984433391563984e-05,
+      "loss": 1.2181,
+      "step": 320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.294366689855624e-05,
+      "loss": 1.2298,
+      "step": 321
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.2902846772544625e-05,
+      "loss": 1.2335,
+      "step": 322
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2861973757263416e-05,
+      "loss": 1.2078,
+      "step": 323
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.282104859741052e-05,
+      "loss": 1.2237,
+      "step": 324
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2780072038633913e-05,
+      "loss": 1.2552,
+      "step": 325
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2739044827518043e-05,
+      "loss": 1.2443,
+      "step": 326
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2697967711570243e-05,
+      "loss": 1.2096,
+      "step": 327
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2656841439207093e-05,
+      "loss": 1.2598,
+      "step": 328
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2615666759740788e-05,
+      "loss": 1.2469,
+      "step": 329
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2574444423365503e-05,
+      "loss": 1.2612,
+      "step": 330
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2533175181143704e-05,
+      "loss": 1.2247,
+      "step": 331
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2491859784992477e-05,
+      "loss": 1.2132,
+      "step": 332
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.245049898766982e-05,
+      "loss": 1.2206,
+      "step": 333
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2409093542760925e-05,
+      "loss": 1.2533,
+      "step": 334
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2367644204664468e-05,
+      "loss": 1.2584,
+      "step": 335
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2326151728578839e-05,
+      "loss": 1.2136,
+      "step": 336
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.228461687048839e-05,
+      "loss": 1.2554,
+      "step": 337
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2243040387149682e-05,
+      "loss": 1.2313,
+      "step": 338
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2201423036077657e-05,
+      "loss": 1.2444,
+      "step": 339
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2159765575531877e-05,
+      "loss": 1.2384,
+      "step": 340
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2118068764502677e-05,
+      "loss": 1.2357,
+      "step": 341
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2076333362697358e-05,
+      "loss": 1.2348,
+      "step": 342
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2034560130526341e-05,
+      "loss": 1.2197,
+      "step": 343
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.199274982908929e-05,
+      "loss": 1.2181,
+      "step": 344
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.1950903220161286e-05,
+      "loss": 1.236,
+      "step": 345
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1909021066178906e-05,
+      "loss": 1.2416,
+      "step": 346
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1867104130226363e-05,
+      "loss": 1.2631,
+      "step": 347
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1825153176021591e-05,
+      "loss": 1.2359,
+      "step": 348
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1783168967902314e-05,
+      "loss": 1.2483,
+      "step": 349
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1741152270812155e-05,
+      "loss": 1.2347,
+      "step": 350
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1699103850286668e-05,
+      "loss": 1.2457,
+      "step": 351
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1657024472439402e-05,
+      "loss": 1.243,
+      "step": 352
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1614914903947952e-05,
+      "loss": 1.2467,
+      "step": 353
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.157277591203996e-05,
+      "loss": 1.254,
+      "step": 354
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.153060826447918e-05,
+      "loss": 1.242,
+      "step": 355
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1488412729551449e-05,
+      "loss": 1.2393,
+      "step": 356
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.144619007605071e-05,
+      "loss": 1.2137,
+      "step": 357
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1403941073265014e-05,
+      "loss": 1.2508,
+      "step": 358
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1361666490962468e-05,
+      "loss": 1.2404,
+      "step": 359
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1319367099377248e-05,
+      "loss": 1.2699,
+      "step": 360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1277043669195549e-05,
+      "loss": 1.2087,
+      "step": 361
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1234696971541534e-05,
+      "loss": 1.2189,
+      "step": 362
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1192327777963313e-05,
+      "loss": 1.2292,
+      "step": 363
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1149936860418846e-05,
+      "loss": 1.2429,
+      "step": 364
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1107524991261913e-05,
+      "loss": 1.2296,
+      "step": 365
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1065092943228024e-05,
+      "loss": 1.2658,
+      "step": 366
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1022641489420342e-05,
+      "loss": 1.2367,
+      "step": 367
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.098017140329561e-05,
+      "loss": 1.2268,
+      "step": 368
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0937683458650029e-05,
+      "loss": 1.2717,
+      "step": 369
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0895178429605189e-05,
+      "loss": 1.2409,
+      "step": 370
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0852657090593961e-05,
+      "loss": 1.2242,
+      "step": 371
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0810120216346368e-05,
+      "loss": 1.2232,
+      "step": 372
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0767568581875494e-05,
+      "loss": 1.2814,
+      "step": 373
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.072500296246334e-05,
+      "loss": 1.2448,
+      "step": 374
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0682424133646712e-05,
+      "loss": 1.2454,
+      "step": 375
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0639832871203094e-05,
+      "loss": 1.2136,
+      "step": 376
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0597229951136498e-05,
+      "loss": 1.222,
+      "step": 377
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0554616149663355e-05,
+      "loss": 1.2472,
+      "step": 378
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0511992243198335e-05,
+      "loss": 1.2139,
+      "step": 379
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0469359008340216e-05,
+      "loss": 1.2283,
+      "step": 380
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0426717221857756e-05,
+      "loss": 1.2242,
+      "step": 381
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0384067660675508e-05,
+      "loss": 1.2512,
+      "step": 382
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.034141110185968e-05,
+      "loss": 1.2419,
+      "step": 383
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0298748322603982e-05,
+      "loss": 1.2473,
+      "step": 384
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0256080100215448e-05,
+      "loss": 1.2309,
+      "step": 385
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0213407212100296e-05,
+      "loss": 1.2165,
+      "step": 386
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.017073043574975e-05,
+      "loss": 1.2257,
+      "step": 387
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0128050548725865e-05,
+      "loss": 1.2333,
+      "step": 388
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0085368328647395e-05,
+      "loss": 1.1892,
+      "step": 389
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0042684553175575e-05,
+      "loss": 1.221,
+      "step": 390
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 1e-05,
+      "loss": 1.2375,
+      "step": 391
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.957315446824425e-06,
+      "loss": 1.2177,
+      "step": 392
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.91463167135261e-06,
+      "loss": 1.2274,
+      "step": 393
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.871949451274137e-06,
+      "loss": 1.2043,
+      "step": 394
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.829269564250254e-06,
+      "loss": 1.2204,
+      "step": 395
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.786592787899707e-06,
+      "loss": 1.2835,
+      "step": 396
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.743919899784555e-06,
+      "loss": 1.2021,
+      "step": 397
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.701251677396021e-06,
+      "loss": 1.2067,
+      "step": 398
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.658588898140322e-06,
+      "loss": 1.2337,
+      "step": 399
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.615932339324497e-06,
+      "loss": 1.2177,
+      "step": 400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.573282778142246e-06,
+      "loss": 1.2336,
+      "step": 401
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.530640991659785e-06,
+      "loss": 1.227,
+      "step": 402
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.488007756801672e-06,
+      "loss": 1.2225,
+      "step": 403
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.445383850336648e-06,
+      "loss": 1.2252,
+      "step": 404
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.402770048863502e-06,
+      "loss": 1.1976,
+      "step": 405
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.360167128796913e-06,
+      "loss": 1.2518,
+      "step": 406
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.317575866353293e-06,
+      "loss": 1.2183,
+      "step": 407
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.274997037536663e-06,
+      "loss": 1.2364,
+      "step": 408
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.232431418124507e-06,
+      "loss": 1.2687,
+      "step": 409
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.189879783653633e-06,
+      "loss": 1.2287,
+      "step": 410
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.14734290940604e-06,
+      "loss": 1.2357,
+      "step": 411
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.104821570394811e-06,
+      "loss": 1.2428,
+      "step": 412
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.062316541349978e-06,
+      "loss": 1.2327,
+      "step": 413
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.019828596704394e-06,
+      "loss": 1.2187,
+      "step": 414
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.977358510579658e-06,
+      "loss": 1.2402,
+      "step": 415
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.93490705677198e-06,
+      "loss": 1.2443,
+      "step": 416
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.89247500873809e-06,
+      "loss": 1.2737,
+      "step": 417
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.850063139581156e-06,
+      "loss": 1.2073,
+      "step": 418
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.807672222036692e-06,
+      "loss": 1.2182,
+      "step": 419
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.765303028458468e-06,
+      "loss": 1.2306,
+      "step": 420
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.722956330804456e-06,
+      "loss": 1.2609,
+      "step": 421
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.680632900622752e-06,
+      "loss": 1.1832,
+      "step": 422
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.638333509037537e-06,
+      "loss": 1.2261,
+      "step": 423
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.59605892673499e-06,
+      "loss": 1.2133,
+      "step": 424
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.55380992394929e-06,
+      "loss": 1.2271,
+      "step": 425
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.511587270448556e-06,
+      "loss": 1.2241,
+      "step": 426
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.469391735520824e-06,
+      "loss": 1.2091,
+      "step": 427
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.42722408796004e-06,
+      "loss": 1.247,
+      "step": 428
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.385085096052053e-06,
+      "loss": 1.2601,
+      "step": 429
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.342975527560601e-06,
+      "loss": 1.2767,
+      "step": 430
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.300896149713334e-06,
+      "loss": 1.2175,
+      "step": 431
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.258847729187845e-06,
+      "loss": 1.2192,
+      "step": 432
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.216831032097689e-06,
+      "loss": 1.2512,
+      "step": 433
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.174846823978412e-06,
+      "loss": 1.2384,
+      "step": 434
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.132895869773638e-06,
+      "loss": 1.2098,
+      "step": 435
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.0909789338211e-06,
+      "loss": 1.2061,
+      "step": 436
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.04909677983872e-06,
+      "loss": 1.2013,
+      "step": 437
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.00725017091071e-06,
+      "loss": 1.1785,
+      "step": 438
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.965439869473664e-06,
+      "loss": 1.2051,
+      "step": 439
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.923666637302643e-06,
+      "loss": 1.2502,
+      "step": 440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.881931235497324e-06,
+      "loss": 1.1977,
+      "step": 441
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.84023442446813e-06,
+      "loss": 1.1519,
+      "step": 442
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.798576963922347e-06,
+      "loss": 1.2102,
+      "step": 443
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.75695961285032e-06,
+      "loss": 1.2243,
+      "step": 444
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.71538312951161e-06,
+      "loss": 1.2375,
+      "step": 445
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.673848271421166e-06,
+      "loss": 1.2186,
+      "step": 446
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.632355795335533e-06,
+      "loss": 1.2363,
+      "step": 447
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.590906457239073e-06,
+      "loss": 1.2394,
+      "step": 448
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.549501012330184e-06,
+      "loss": 1.214,
+      "step": 449
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.508140215007526e-06,
+      "loss": 1.2012,
+      "step": 450
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.466824818856296e-06,
+      "loss": 1.2206,
+      "step": 451
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.4255555766345025e-06,
+      "loss": 1.2179,
+      "step": 452
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.384333240259216e-06,
+      "loss": 1.2296,
+      "step": 453
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.34315856079291e-06,
+      "loss": 1.1944,
+      "step": 454
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.3020322884297565e-06,
+      "loss": 1.2263,
+      "step": 455
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.260955172481959e-06,
+      "loss": 1.2106,
+      "step": 456
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.219927961366091e-06,
+      "loss": 1.2275,
+      "step": 457
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.178951402589482e-06,
+      "loss": 1.1832,
+      "step": 458
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.1380262427365885e-06,
+      "loss": 1.2118,
+      "step": 459
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.097153227455379e-06,
+      "loss": 1.2205,
+      "step": 460
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.056333101443761e-06,
+      "loss": 1.2179,
+      "step": 461
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.01556660843602e-06,
+      "loss": 1.2121,
+      "step": 462
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 6.974854491189243e-06,
+      "loss": 1.2206,
+      "step": 463
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 6.934197491469818e-06,
+      "loss": 1.24,
+      "step": 464
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 6.893596350039896e-06,
+      "loss": 1.204,
+      "step": 465
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 6.853051806643898e-06,
+      "loss": 1.2224,
+      "step": 466
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.812564599995042e-06,
+      "loss": 1.227,
+      "step": 467
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.772135467761889e-06,
+      "loss": 1.2361,
+      "step": 468
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.731765146554891e-06,
+      "loss": 1.2099,
+      "step": 469
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.691454371912974e-06,
+      "loss": 1.1969,
+      "step": 470
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.651203878290139e-06,
+      "loss": 1.2269,
+      "step": 471
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.6110143990420824e-06,
+      "loss": 1.2106,
+      "step": 472
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.570886666412823e-06,
+      "loss": 1.1969,
+      "step": 473
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.5308214115213785e-06,
+      "loss": 1.239,
+      "step": 474
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.490819364348434e-06,
+      "loss": 1.2399,
+      "step": 475
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.450881253723035e-06,
+      "loss": 1.2265,
+      "step": 476
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.41100780730932e-06,
+      "loss": 1.2446,
+      "step": 477
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.371199751593264e-06,
+      "loss": 1.2229,
+      "step": 478
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.331457811869437e-06,
+      "loss": 1.2036,
+      "step": 479
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.291782712227776e-06,
+      "loss": 1.2002,
+      "step": 480
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.2521751755404226e-06,
+      "loss": 1.2262,
+      "step": 481
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.212635923448526e-06,
+      "loss": 1.2469,
+      "step": 482
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.173165676349103e-06,
+      "loss": 1.2008,
+      "step": 483
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.133765153381918e-06,
+      "loss": 1.2352,
+      "step": 484
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.094435072416379e-06,
+      "loss": 1.2257,
+      "step": 485
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.055176150038445e-06,
+      "loss": 1.2022,
+      "step": 486
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.015989101537586e-06,
+      "loss": 1.2565,
+      "step": 487
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 5.976874640893751e-06,
+      "loss": 1.2518,
+      "step": 488
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 5.937833480764339e-06,
+      "loss": 1.2272,
+      "step": 489
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.898866332471241e-06,
+      "loss": 1.2077,
+      "step": 490
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.859973905987866e-06,
+      "loss": 1.2364,
+      "step": 491
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.821156909926202e-06,
+      "loss": 1.2377,
+      "step": 492
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.782416051523909e-06,
+      "loss": 1.2399,
+      "step": 493
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.743752036631443e-06,
+      "loss": 1.192,
+      "step": 494
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.7051655696991825e-06,
+      "loss": 1.242,
+      "step": 495
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.666657353764594e-06,
+      "loss": 1.2175,
+      "step": 496
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.628228090439434e-06,
+      "loss": 1.2376,
+      "step": 497
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.589878479896959e-06,
+      "loss": 1.2259,
+      "step": 498
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.55160922085916e-06,
+      "loss": 1.232,
+      "step": 499
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.513421010584044e-06,
+      "loss": 1.2215,
+      "step": 500
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.4753145448529284e-06,
+      "loss": 1.2117,
+      "step": 501
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.437290517957767e-06,
+      "loss": 1.1955,
+      "step": 502
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.399349622688479e-06,
+      "loss": 1.2329,
+      "step": 503
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.3614925503203586e-06,
+      "loss": 1.2339,
+      "step": 504
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.323719990601459e-06,
+      "loss": 1.2062,
+      "step": 505
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.286032631740023e-06,
+      "loss": 1.2327,
+      "step": 506
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.248431160391963e-06,
+      "loss": 1.2644,
+      "step": 507
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.2109162616483325e-06,
+      "loss": 1.194,
+      "step": 508
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.1734886190228496e-06,
+      "loss": 1.2536,
+      "step": 509
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.136148914439441e-06,
+      "loss": 1.2289,
+      "step": 510
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.098897828219831e-06,
+      "loss": 1.2126,
+      "step": 511
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.061736039071124e-06,
+      "loss": 1.188,
+      "step": 512
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 5.024664224073454e-06,
+      "loss": 1.2643,
+      "step": 513
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.987683058667651e-06,
+      "loss": 1.2361,
+      "step": 514
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.950793216642923e-06,
+      "loss": 1.2565,
+      "step": 515
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.913995370124578e-06,
+      "loss": 1.2317,
+      "step": 516
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.877290189561795e-06,
+      "loss": 1.2132,
+      "step": 517
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.840678343715399e-06,
+      "loss": 1.214,
+      "step": 518
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.804160499645667e-06,
+      "loss": 1.2085,
+      "step": 519
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.767737322700185e-06,
+      "loss": 1.2343,
+      "step": 520
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.7314094765017325e-06,
+      "loss": 1.2287,
+      "step": 521
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.695177622936169e-06,
+      "loss": 1.2358,
+      "step": 522
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.659042422140399e-06,
+      "loss": 1.2439,
+      "step": 523
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.623004532490328e-06,
+      "loss": 1.2239,
+      "step": 524
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.587064610588881e-06,
+      "loss": 1.2292,
+      "step": 525
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.551223311254013e-06,
+      "loss": 1.227,
+      "step": 526
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.515481287506811e-06,
+      "loss": 1.2171,
+      "step": 527
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.479839190559583e-06,
+      "loss": 1.2125,
+      "step": 528
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.444297669803981e-06,
+      "loss": 1.2338,
+      "step": 529
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.408857372799179e-06,
+      "loss": 1.2236,
+      "step": 530
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.37351894526009e-06,
+      "loss": 1.2755,
+      "step": 531
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.338283031045567e-06,
+      "loss": 1.2603,
+      "step": 532
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.303150272146706e-06,
+      "loss": 1.2462,
+      "step": 533
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.268121308675132e-06,
+      "loss": 1.2025,
+      "step": 534
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.2331967788513295e-06,
+      "loss": 1.231,
+      "step": 535
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.198377318993035e-06,
+      "loss": 1.221,
+      "step": 536
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.1636635635036235e-06,
+      "loss": 1.2132,
+      "step": 537
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.129056144860567e-06,
+      "loss": 1.2579,
+      "step": 538
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.094555693603891e-06,
+      "loss": 1.2362,
+      "step": 539
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.060162838324708e-06,
+      "loss": 1.2067,
+      "step": 540
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.025878205653747e-06,
+      "loss": 1.2212,
+      "step": 541
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 3.991702420249941e-06,
+      "loss": 1.2176,
+      "step": 542
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.957636104789056e-06,
+      "loss": 1.2189,
+      "step": 543
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.9236798799523375e-06,
+      "loss": 1.204,
+      "step": 544
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.8898343644151945e-06,
+      "loss": 1.2195,
+      "step": 545
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.856100174835945e-06,
+      "loss": 1.2191,
+      "step": 546
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.822477925844564e-06,
+      "loss": 1.2254,
+      "step": 547
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.7889682300315e-06,
+      "loss": 1.2103,
+      "step": 548
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.755571697936493e-06,
+      "loss": 1.2557,
+      "step": 549
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.722288938037478e-06,
+      "loss": 1.2157,
+      "step": 550
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.689120556739475e-06,
+      "loss": 1.2242,
+      "step": 551
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.6560671583635467e-06,
+      "loss": 1.2322,
+      "step": 552
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.6231293451357994e-06,
+      "loss": 1.1993,
+      "step": 553
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.590307717176401e-06,
+      "loss": 1.2278,
+      "step": 554
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.557602872488638e-06,
+      "loss": 1.2176,
+      "step": 555
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.525015406948039e-06,
+      "loss": 1.2265,
+      "step": 556
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.492545914291512e-06,
+      "loss": 1.2085,
+      "step": 557
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.4601949861065086e-06,
+      "loss": 1.2287,
+      "step": 558
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.4279632118202744e-06,
+      "loss": 1.2073,
+      "step": 559
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.3958511786890923e-06,
+      "loss": 1.2092,
+      "step": 560
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.3638594717875807e-06,
+      "loss": 1.2181,
+      "step": 561
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.33198867399804e-06,
+      "loss": 1.2126,
+      "step": 562
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.3002393659998357e-06,
+      "loss": 1.2117,
+      "step": 563
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.2686121262588165e-06,
+      "loss": 1.2126,
+      "step": 564
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.2371075310167634e-06,
+      "loss": 1.226,
+      "step": 565
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.205726154280905e-06,
+      "loss": 1.1847,
+      "step": 566
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.174468567813461e-06,
+      "loss": 1.2131,
+      "step": 567
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.143335341121202e-06,
+      "loss": 1.2046,
+      "step": 568
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.1123270414451035e-06,
+      "loss": 1.2495,
+      "step": 569
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.081444233749994e-06,
+      "loss": 1.2398,
+      "step": 570
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.050687480714256e-06,
+      "loss": 1.2245,
+      "step": 571
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.0200573427195877e-06,
+      "loss": 1.2042,
+      "step": 572
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.9895543778407875e-06,
+      "loss": 1.2298,
+      "step": 573
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.959179141835591e-06,
+      "loss": 1.2284,
+      "step": 574
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.9289321881345257e-06,
+      "loss": 1.2351,
+      "step": 575
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.898814067830855e-06,
+      "loss": 1.2263,
+      "step": 576
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.868825329670524e-06,
+      "loss": 1.2225,
+      "step": 577
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.83896652004215e-06,
+      "loss": 1.2288,
+      "step": 578
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.809238182967092e-06,
+      "loss": 1.2179,
+      "step": 579
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.779640860089523e-06,
+      "loss": 1.2299,
+      "step": 580
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.7501750906665603e-06,
+      "loss": 1.1786,
+      "step": 581
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.7208414115584436e-06,
+      "loss": 1.229,
+      "step": 582
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.691640357218759e-06,
+      "loss": 1.2126,
+      "step": 583
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.662572459684699e-06,
+      "loss": 1.2048,
+      "step": 584
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.6336382485673574e-06,
+      "loss": 1.2251,
+      "step": 585
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.6048382510420954e-06,
+      "loss": 1.2086,
+      "step": 586
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.576172991838933e-06,
+      "loss": 1.2318,
+      "step": 587
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.547642993232976e-06,
+      "loss": 1.2496,
+      "step": 588
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.519248775034918e-06,
+      "loss": 1.2442,
+      "step": 589
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.490990854581563e-06,
+      "loss": 1.2059,
+      "step": 590
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.4628697467263916e-06,
+      "loss": 1.2276,
+      "step": 591
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.4348859638301857e-06,
+      "loss": 1.2529,
+      "step": 592
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.4070400157517036e-06,
+      "loss": 1.209,
+      "step": 593
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.3793324098383796e-06,
+      "loss": 1.2104,
+      "step": 594
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.351763650917074e-06,
+      "loss": 1.2115,
+      "step": 595
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.3243342412848923e-06,
+      "loss": 1.1976,
+      "step": 596
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.2970446807000237e-06,
+      "loss": 1.225,
+      "step": 597
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.26989546637263e-06,
+      "loss": 1.226,
+      "step": 598
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.2428870929558012e-06,
+      "loss": 1.2215,
+      "step": 599
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.2160200525365326e-06,
+      "loss": 1.2065,
+      "step": 600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1892948346267583e-06,
+      "loss": 1.244,
+      "step": 601
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1627119261544348e-06,
+      "loss": 1.2354,
+      "step": 602
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1362718114546777e-06,
+      "loss": 1.257,
+      "step": 603
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.109974972260921e-06,
+      "loss": 1.2592,
+      "step": 604
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0838218876961524e-06,
+      "loss": 1.2424,
+      "step": 605
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.057813034264181e-06,
+      "loss": 1.2211,
+      "step": 606
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0319488858409552e-06,
+      "loss": 1.2308,
+      "step": 607
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0062299136659203e-06,
+      "loss": 1.2381,
+      "step": 608
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.980656586333449e-06,
+      "loss": 1.2114,
+      "step": 609
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.955229369784295e-06,
+      "loss": 1.2086,
+      "step": 610
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.929948727297096e-06,
+      "loss": 1.2207,
+      "step": 611
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9048151194799435e-06,
+      "loss": 1.1851,
+      "step": 612
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8798290042619949e-06,
+      "loss": 1.2128,
+      "step": 613
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8549908368851099e-06,
+      "loss": 1.2371,
+      "step": 614
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8303010698955803e-06,
+      "loss": 1.208,
+      "step": 615
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8057601531358693e-06,
+      "loss": 1.2582,
+      "step": 616
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.7813685337364205e-06,
+      "loss": 1.2378,
+      "step": 617
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.7571266561075073e-06,
+      "loss": 1.2273,
+      "step": 618
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7330349619311415e-06,
+      "loss": 1.2149,
+      "step": 619
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7090938901530264e-06,
+      "loss": 1.2558,
+      "step": 620
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.6853038769745466e-06,
+      "loss": 1.1991,
+      "step": 621
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.6616653558448437e-06,
+      "loss": 1.2226,
+      "step": 622
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.638178757452894e-06,
+      "loss": 1.2173,
+      "step": 623
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.614844509719674e-06,
+      "loss": 1.2116,
+      "step": 624
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.5916630377903696e-06,
+      "loss": 1.2148,
+      "step": 625
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.5686347640266208e-06,
+      "loss": 1.1965,
+      "step": 626
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.5457601079988226e-06,
+      "loss": 1.2602,
+      "step": 627
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.5230394864784925e-06,
+      "loss": 1.1897,
+      "step": 628
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.5004733134306692e-06,
+      "loss": 1.2226,
+      "step": 629
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.478062000006375e-06,
+      "loss": 1.2064,
+      "step": 630
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.4558059545351144e-06,
+      "loss": 1.217,
+      "step": 631
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.4337055825174506e-06,
+      "loss": 1.1877,
+      "step": 632
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.4117612866176022e-06,
+      "loss": 1.2036,
+      "step": 633
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.3899734666561138e-06,
+      "loss": 1.2212,
+      "step": 634
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.3683425196025734e-06,
+      "loss": 1.2119,
+      "step": 635
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.3468688395683783e-06,
+      "loss": 1.2475,
+      "step": 636
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.325552817799547e-06,
+      "loss": 1.2041,
+      "step": 637
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.3043948426696019e-06,
+      "loss": 1.2492,
+      "step": 638
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.2833952996724864e-06,
+      "loss": 1.2403,
+      "step": 639
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.2625545714155474e-06,
+      "loss": 1.2125,
+      "step": 640
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.24187303761255e-06,
+      "loss": 1.2063,
+      "step": 641
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.221351075076781e-06,
+      "loss": 1.2495,
+      "step": 642
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.2009890577141625e-06,
+      "loss": 1.2197,
+      "step": 643
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.1807873565164507e-06,
+      "loss": 1.2521,
+      "step": 644
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.1607463395544782e-06,
+      "loss": 1.2254,
+      "step": 645
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.1408663719714418e-06,
+      "loss": 1.2357,
+      "step": 646
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.121147815976248e-06,
+      "loss": 1.1756,
+      "step": 647
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.1015910308369239e-06,
+      "loss": 1.1827,
+      "step": 648
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.0821963728740626e-06,
+      "loss": 1.2071,
+      "step": 649
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.062964195454329e-06,
+      "loss": 1.2552,
+      "step": 650
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.0438948489840327e-06,
+      "loss": 1.1921,
+      "step": 651
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.0249886809027355e-06,
+      "loss": 1.2148,
+      "step": 652
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.0062460356769189e-06,
+      "loss": 1.2207,
+      "step": 653
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 9.876672547937117e-07,
+      "loss": 1.1999,
+      "step": 654
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 9.692526767546727e-07,
+      "loss": 1.2209,
+      "step": 655
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 9.51002637069619e-07,
+      "loss": 1.2208,
+      "step": 656
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 9.32917468250506e-07,
+      "loss": 1.2426,
+      "step": 657
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 9.149974998053823e-07,
+      "loss": 1.2129,
+      "step": 658
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.972430582323788e-07,
+      "loss": 1.2453,
+      "step": 659
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.796544670137574e-07,
+      "loss": 1.1991,
+      "step": 660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.622320466100242e-07,
+      "loss": 1.2346,
+      "step": 661
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.449761144540869e-07,
+      "loss": 1.2006,
+      "step": 662
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.278869849454718e-07,
+      "loss": 1.2171,
+      "step": 663
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.109649694445898e-07,
+      "loss": 1.2201,
+      "step": 664
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.942103762670783e-07,
+      "loss": 1.2213,
+      "step": 665
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.776235106781704e-07,
+      "loss": 1.2146,
+      "step": 666
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.612046748871327e-07,
+      "loss": 1.2159,
+      "step": 667
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.449541680417704e-07,
+      "loss": 1.2434,
+      "step": 668
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.288722862229691e-07,
+      "loss": 1.1792,
+      "step": 669
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.12959322439295e-07,
+      "loss": 1.2433,
+      "step": 670
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.972155666216684e-07,
+      "loss": 1.2162,
+      "step": 671
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.816413056180748e-07,
+      "loss": 1.2319,
+      "step": 672
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.662368231883388e-07,
+      "loss": 1.2424,
+      "step": 673
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.510023999989501e-07,
+      "loss": 1.2052,
+      "step": 674
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.359383136179598e-07,
+      "loss": 1.2292,
+      "step": 675
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.210448385099177e-07,
+      "loss": 1.2327,
+      "step": 676
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.063222460308649e-07,
+      "loss": 1.2206,
+      "step": 677
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.917708044234017e-07,
+      "loss": 1.2348,
+      "step": 678
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.77390778811796e-07,
+      "loss": 1.2034,
+      "step": 679
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.631824311971456e-07,
+      "loss": 1.2399,
+      "step": 680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.491460204526156e-07,
+      "loss": 1.1637,
+      "step": 681
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.352818023187167e-07,
+      "loss": 1.1979,
+      "step": 682
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.215900293986431e-07,
+      "loss": 1.233,
+      "step": 683
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.08070951153673e-07,
+      "loss": 1.2375,
+      "step": 684
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.947248138986249e-07,
+      "loss": 1.2333,
+      "step": 685
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.81551860797369e-07,
+      "loss": 1.2321,
+      "step": 686
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6855233185839175e-07,
+      "loss": 1.2235,
+      "step": 687
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.557264639304315e-07,
+      "loss": 1.2285,
+      "step": 688
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.430744906981577e-07,
+      "loss": 1.2414,
+      "step": 689
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.305966426779118e-07,
+      "loss": 1.2246,
+      "step": 690
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.1829314721351213e-07,
+      "loss": 1.23,
+      "step": 691
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.0616422847211013e-07,
+      "loss": 1.2381,
+      "step": 692
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.942101074401028e-07,
+      "loss": 1.2074,
+      "step": 693
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.824310019191102e-07,
+      "loss": 1.1853,
+      "step": 694
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.708271265220087e-07,
+      "loss": 1.2447,
+      "step": 695
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.5939869266901073e-07,
+      "loss": 1.2242,
+      "step": 696
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.481459085838268e-07,
+      "loss": 1.2555,
+      "step": 697
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.370689792898618e-07,
+      "loss": 1.2489,
+      "step": 698
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.261681066064859e-07,
+      "loss": 1.242,
+      "step": 699
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.154434891453473e-07,
+      "loss": 1.2281,
+      "step": 700
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.0489532230676744e-07,
+      "loss": 1.1931,
+      "step": 701
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 2.945237982761706e-07,
+      "loss": 1.189,
+      "step": 702
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.843291060205855e-07,
+      "loss": 1.2069,
+      "step": 703
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.7431143128520243e-07,
+      "loss": 1.1773,
+      "step": 704
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.6447095658999054e-07,
+      "loss": 1.2184,
+      "step": 705
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.5480786122636713e-07,
+      "loss": 1.2064,
+      "step": 706
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.453223212539391e-07,
+      "loss": 1.2445,
+      "step": 707
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.3601450949728876e-07,
+      "loss": 1.1938,
+      "step": 708
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.2688459554282673e-07,
+      "loss": 1.1684,
+      "step": 709
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.1793274573570166e-07,
+      "loss": 1.2272,
+      "step": 710
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.091591231767709e-07,
+      "loss": 1.2135,
+      "step": 711
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.005638877196303e-07,
+      "loss": 1.2497,
+      "step": 712
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.921471959676957e-07,
+      "loss": 1.1679,
+      "step": 713
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.8390920127135613e-07,
+      "loss": 1.2072,
+      "step": 714
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.7585005372517504e-07,
+      "loss": 1.2149,
+      "step": 715
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.6796990016515914e-07,
+      "loss": 1.2245,
+      "step": 716
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.6026888416608267e-07,
+      "loss": 1.2107,
+      "step": 717
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.5274714603886742e-07,
+      "loss": 1.2172,
+      "step": 718
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.4540482282803136e-07,
+      "loss": 1.2217,
+      "step": 719
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.3824204830918952e-07,
+      "loss": 1.235,
+      "step": 720
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.3125895298661705e-07,
+      "loss": 1.228,
+      "step": 721
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.244556640908712e-07,
+      "loss": 1.2113,
+      "step": 722
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.1783230557647075e-07,
+      "loss": 1.1956,
+      "step": 723
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.1138899811964477e-07,
+      "loss": 1.1961,
+      "step": 724
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.0512585911612416e-07,
+      "loss": 1.2284,
+      "step": 725
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 9.904300267901012e-08,
+      "loss": 1.2261,
+      "step": 726
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 9.314053963669245e-08,
+      "loss": 1.2033,
+      "step": 727
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 8.741857753083228e-08,
+      "loss": 1.228,
+      "step": 728
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 8.187722061439806e-08,
+      "loss": 1.2217,
+      "step": 729
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 7.651656984977051e-08,
+      "loss": 1.2252,
+      "step": 730
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 7.133672290690064e-08,
+      "loss": 1.2155,
+      "step": 731
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 6.633777416153232e-08,
+      "loss": 1.2393,
+      "step": 732
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 6.151981469348034e-08,
+      "loss": 1.2009,
+      "step": 733
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 5.688293228497399e-08,
+      "loss": 1.2215,
+      "step": 734
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 5.2427211419051605e-08,
+      "loss": 1.2348,
+      "step": 735
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 4.815273327803183e-08,
+      "loss": 1.2593,
+      "step": 736
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 4.405957574202147e-08,
+      "loss": 1.2245,
+      "step": 737
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 4.014781338751106e-08,
+      "loss": 1.1899,
+      "step": 738
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.641751748600042e-08,
+      "loss": 1.2334,
+      "step": 739
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.2868756002712997e-08,
+      "loss": 1.2141,
+      "step": 740
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.950159359535132e-08,
+      "loss": 1.2478,
+      "step": 741
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.6316091612920146e-08,
+      "loss": 1.2436,
+      "step": 742
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3312308094607382e-08,
+      "loss": 1.2141,
+      "step": 743
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.049029776873268e-08,
+      "loss": 1.1924,
+      "step": 744
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7850112051738255e-08,
+      "loss": 1.2596,
+      "step": 745
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5391799047266287e-08,
+      "loss": 1.1749,
+      "step": 746
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.3115403545270744e-08,
+      "loss": 1.2327,
+      "step": 747
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1020967021210249e-08,
+      "loss": 1.2044,
+      "step": 748
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.108527635284248e-09,
+      "loss": 1.2405,
+      "step": 749
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.378120231745778e-09,
+      "loss": 1.1996,
+      "step": 750
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.8297763382597625e-09,
+      "loss": 1.2085,
+      "step": 751
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.463524165333466e-09,
+      "loss": 1.2112,
+      "step": 752
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2793886057991277e-09,
+      "loss": 1.2329,
+      "step": 753
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.277391234363213e-09,
+      "loss": 1.1981,
+      "step": 754
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4575503072100649e-09,
+      "loss": 1.2015,
+      "step": 755
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.198807616732752e-10,
+      "loss": 1.2342,
+      "step": 756
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.6439421595924065e-10,
+      "loss": 1.2115,
+      "step": 757
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.109896894066161e-11,
+      "loss": 1.2319,
+      "step": 758
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.0,
+      "loss": 1.2129,
+      "step": 759
+    },
+    {
+      "epoch": 1.0,
+      "step": 759,
+      "total_flos": 8.491785698085622e+21,
+      "train_loss": 1.2714912057394097,
+      "train_runtime": 6445.9977,
+      "train_samples_per_second": 30.144,
+      "train_steps_per_second": 0.118
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 759,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "total_flos": 8.491785698085622e+21,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

Pretrain_2B/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36ee2f39aa434a2d5954fd76bde4cbb96331424e75cc0a0b15ea65fced242d53
+size 5819