Training in progress, step 500

all_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 4.998736842105263,
+    "eval_accuracy": 0.42753137093133137,
+    "eval_bleu": 0.14672943718286596,
+    "eval_loss": 3.099884033203125,
+    "eval_perplexity": 22.195377205390898,
+    "eval_runtime": 10.965,
+    "eval_samples": 1141,
+    "eval_samples_per_second": 104.059,
+    "eval_steps_per_second": 6.566,
+    "perplexity": 22.195377205390898,
+    "total_flos": 1.0584067483285586e+18,
+    "train_loss": 3.584141966119902,
+    "train_runtime": 28526.973,
+    "train_samples_per_second": 19.98,
+    "train_steps_per_second": 0.624
+}

config.json

@@ -4,7 +4,12 @@
     "ParallelGPT2LMHeadModel"
   ],
   "attn_pdrop": 0.1,
+  "auto_map": {
+    "AutoConfig": "modeling_parallel_gpt2.ParallelGPT2Config",
+    "AutoModel": "modeling_parallel_gpt2.ParallelGPT2LMHeadModel"
+  },
   "bos_token_id": 50256,
+  "bottleneck_method": "concat",
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,

eval_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 4.998736842105263,
+    "eval_accuracy": 0.42753137093133137,
+    "eval_bleu": 0.14672943718286596,
+    "eval_loss": 3.099884033203125,
+    "eval_perplexity": 22.195377205390898,
+    "eval_runtime": 10.965,
+    "eval_samples": 1141,
+    "eval_samples_per_second": 104.059,
+    "eval_steps_per_second": 6.566,
+    "perplexity": 22.195377205390898
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a821777bb11dfd632198f14ae0c0ce726eaececac5fb4323a5dffe1c426a28e
-size 1419322880
+oid sha256:4d42579b7903b46b75f5aecc1f943832ad871b6383666ceb81c3a200b23dcd3f
+size 1427715776

modeling_parallel_gpt2.py

@@ -0,0 +1,496 @@
+
+"""PyTorch OpenAI GPT-2 model modified to support parallel-gpt2, code copied from Huggingface"""
+
+
+import warnings
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPastAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions
+)
+from transformers.generation import GenerationMixin
+from transformers.utils.model_parallel_utils import assert_device_map, get_device_map
+from src.models.modeling_gpt2 import GPT2PreTrainedModel, GPT2Block
+from transformers.models.gpt2.configuration_gpt2 import GPT2Config
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa
+
+class ParallelGPT2Config(GPT2Config):
+    model_type = "parallel-gpt2"
+    architectures = ["ParallelGPT2LMHeadModel"]
+
+class ParallelGPT2PretrainedModel(GPT2PreTrainedModel):
+    config_class = ParallelGPT2Config
+
+class ParallelGPT2Model(ParallelGPT2PretrainedModel):
+    _supports_param_buffer_assignment = False
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.embed_dim = config.hidden_size
+
+        self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
+        self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
+
+        self.drop = nn.Dropout(config.embd_pdrop)
+        if config.num_hidden_layers % 2 != 0:
+            raise ValueError("Number of hidden layers must be even")
+        self.h = nn.ModuleList([GPT2Block(config, layer_idx=i) for i in range(config.num_hidden_layers)])
+        self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
+        self.config.bottleneck_method = getattr(config, "bottleneck_method", "mean")
+        if self.config.bottleneck_method=="concate":
+            self.bottleneck = nn.Linear(2*self.embed_dim, self.embed_dim)
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+        self.gradient_checkpointing = False
+        self._attn_implementation = config._attn_implementation
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+
+    def parallelize(self, device_map=None):
+        # Check validity of device_map
+        warnings.warn(
+            "`GPT2Model.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your"
+            " model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own"
+            " `device_map` but it needs to be a dictionary module_name to device, so for instance {'h.0': 0, 'h.1': 1,"
+            " ...}",
+            FutureWarning,
+        )
+        self.device_map = (
+            get_device_map(len(self.h), range(torch.cuda.device_count())) if device_map is None else device_map
+        )
+        assert_device_map(self.device_map, len(self.h))
+        self.model_parallel = True
+        self.first_device = "cpu" if "cpu" in self.device_map.keys() else "cuda:" + str(min(self.device_map.keys()))
+        self.last_device = "cuda:" + str(max(self.device_map.keys()))
+        self.wte = self.wte.to(self.first_device)
+        self.wpe = self.wpe.to(self.first_device)
+        # Load onto devices
+        for k, v in self.device_map.items():
+            for block in v:
+                cuda_device = "cuda:" + str(k)
+                self.h[block] = self.h[block].to(cuda_device)
+        # ln_f to last
+        self.ln_f = self.ln_f.to(self.last_device)
+
+    def deparallelize(self):
+        self.model_parallel = False
+        self.device_map = None
+        self.first_device = "cpu"
+        self.last_device = "cpu"
+        self.wte = self.wte.to("cpu")
+        self.wpe = self.wpe.to("cpu")
+        for index in range(len(self.h)):
+            self.h[index] = self.h[index].to("cpu")
+        self.ln_f = self.ln_f.to("cpu")
+        torch.cuda.empty_cache()
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+
+    def _prune_heads(self, heads_to_prune):
+        """
+        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+        """
+        for layer, heads in heads_to_prune.items():
+            self.h[layer].attn.prune_heads(heads)
+
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+            batch_size = input_ids.shape[0]
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size = inputs_embeds.shape[0]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if token_type_ids is not None:
+            token_type_ids = token_type_ids.view(-1, input_shape[-1])
+
+        if past_key_values is None:
+            past_length = 0
+            past_key_values = tuple([None] * len(self.h))
+        else:
+            past_length = past_key_values[0][0].size(-2)
+        if position_ids is None:
+            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
+        position_embeds = self.wpe(position_ids)
+        hidden_states = inputs_embeds + position_embeds.to(inputs_embeds.device)
+
+        # Attention mask.
+        _use_sdpa = self._attn_implementation == "sdpa" and output_attentions is False and head_mask is None
+        attention_mask = attention_mask.view(batch_size, -1) if attention_mask is not None else None
+        if self._attn_implementation == "flash_attention_2":
+            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
+        elif _use_sdpa:
+            attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
+                attention_mask=attention_mask,
+                input_shape=(batch_size, input_shape[-1]),
+                inputs_embeds=inputs_embeds,
+                past_key_values_length=past_length,
+            )
+        else:
+            if attention_mask is not None:
+                # We create a 3D attention mask from a 2D tensor mask.
+                # Sizes are [batch_size, 1, 1, to_seq_length]
+                # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+                # this attention mask is more simple than the triangular masking of causal attention
+                # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+                attention_mask = attention_mask[:, None, None, :]
+
+                # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+                # masked positions, this operation will create a tensor which is 0.0 for
+                # positions we want to attend and the dtype's smallest value for masked positions.
+                # Since we are adding it to the raw scores before the softmax, this is
+                # effectively the same as removing these entirely.
+                attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+                attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if self.config.add_cross_attention and encoder_hidden_states is not None:
+            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            if encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
+            if _use_sdpa:
+                encoder_attention_mask = _prepare_4d_attention_mask_for_sdpa(
+                    mask=encoder_attention_mask, dtype=inputs_embeds.dtype, tgt_len=input_shape[-1]
+                )
+            elif not self._attn_implementation == "flash_attention_2":
+                encoder_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # head_mask has shape n_layer x batch x n_heads x N x N
+        head_mask = self.get_head_mask(head_mask, self.config.n_layer)
+
+        if token_type_ids is not None:
+            token_type_embeds = self.wte(token_type_ids)
+            hidden_states = hidden_states + token_type_embeds
+
+        hidden_states = self.drop(hidden_states)
+
+        output_shape = (-1,) + input_shape[1:] + (hidden_states.size(-1),)
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        presents = () if use_cache else None
+        all_self_attentions_left = () if output_attentions else None
+        all_self_attentions_right = () if output_attentions else None
+        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
+        all_hidden_states = () if output_hidden_states else None
+        for i in range(0, len(self.h), 2):
+            block_left, layer_past_left = self.h[i], past_key_values[i]
+            block_right, layer_past_right = self.h[i+1], past_key_values[i+1]
+            # Model parallel
+            if self.model_parallel:
+                torch.cuda.set_device(hidden_states.device)
+                # Ensure layer_past is on same device as hidden_states (might not be correct)
+                if layer_past is not None:
+                    layer_past = tuple(past_state.to(hidden_states.device) for past_state in layer_past)
+                # Ensure that attention_mask is always on the same device as hidden_states
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(hidden_states.device)
+                if isinstance(head_mask, torch.Tensor):
+                    head_mask = head_mask.to(hidden_states.device)
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                outputs_left = self._gradient_checkpointing_func(
+                    block_left.__call__,
+                    hidden_states,
+                    None,
+                    attention_mask,
+                    head_mask[i],
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    use_cache,
+                    output_attentions,
+                )
+                outputs_right = self._gradient_checkpointing_func(
+                    block_right.__call__,
+                    hidden_states,
+                    None,
+                    attention_mask,
+                    head_mask[i+1],
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    use_cache,
+                    output_attentions,
+                )
+            else:
+                outputs_left = block_left(
+                    hidden_states,
+                    layer_past=layer_past_left,
+                    attention_mask=attention_mask,
+                    head_mask=head_mask[i],
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_attention_mask,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+                outputs_right = block_right(
+                    hidden_states,
+                    layer_past=layer_past_right,
+                    attention_mask=attention_mask,
+                    head_mask=head_mask[i+1],
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_attention_mask,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+            if self.config.bottleneck_method=="concate":
+                hidden_states = torch.cat((outputs_left[0], outputs_right[0]), dim=-1)
+                hidden_states = self.bottleneck(hidden_states)
+            elif self.config.bottleneck_method=="add":  
+                hidden_states = (outputs_left[0] + outputs_right[0]) ## taking add
+            elif self.config.bottleneck_method=="mean":
+                hidden_states = (outputs_left[0] + outputs_right[0]) / 2 ## taking mean
+            if use_cache is True:
+                presents = presents + (outputs_left[1], outputs_right[1])
+
+            if output_attentions:
+                all_self_attentions_left = all_self_attentions_left + (outputs_left[2 if use_cache else 1],)
+                all_self_attentions_right = all_self_attentions_right + (outputs_right[2 if use_cache else 1],)
+                if self.config.add_cross_attention:
+                    all_cross_attentions_left = all_cross_attentions_left + (outputs_left[3 if use_cache else 2],)
+                    all_cross_attentions_right = all_cross_attentions_right + (outputs_right[3 if use_cache else 2],)
+
+            # Model Parallel: If it's the last layer for that device, put things on the next device
+            if self.model_parallel:
+                for k, v in self.device_map.items():
+                    if i == v[-1] and "cuda:" + str(k) != self.last_device:
+                        hidden_states = hidden_states.to("cuda:" + str(k + 1))
+
+        hidden_states = self.ln_f(hidden_states)
+
+        hidden_states = hidden_states.view(output_shape)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, presents, all_hidden_states, all_self_attentions_left, all_cross_attentions]
+                if v is not None
+            )
+
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions_left,
+            cross_attentions=all_cross_attentions,
+        )
+
+
+class ParallelGPT2LMHeadModel(ParallelGPT2PretrainedModel, GenerationMixin):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.transformer = ParallelGPT2Model(config)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def parallelize(self, device_map=None):
+        warnings.warn(
+            "`GPT2LMHeadModel.parallelize` is deprecated and will be removed in v5 of Transformers, you should load"
+            " your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own"
+            " `device_map` but it needs to be a dictionary module_name to device, so for instance {'transformer.h.0':"
+            " 0, 'transformer.h.1': 1, ...}",
+            FutureWarning,
+        )
+        self.device_map = (
+            get_device_map(len(self.transformer.h), range(torch.cuda.device_count()))
+            if device_map is None
+            else device_map
+        )
+        assert_device_map(self.device_map, len(self.transformer.h))
+        self.transformer.parallelize(self.device_map)
+        self.lm_head = self.lm_head.to(self.transformer.first_device)
+        self.model_parallel = True
+
+    def deparallelize(self):
+        self.transformer.deparallelize()
+        self.transformer = self.transformer.to("cpu")
+        self.lm_head = self.lm_head.to("cpu")
+        self.model_parallel = False
+        torch.cuda.empty_cache()
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
+            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+
+        # Set device for model parallelism
+        if self.model_parallel:
+            torch.cuda.set_device(self.transformer.first_device)
+            hidden_states = hidden_states.to(self.lm_head.weight.device)
+
+        lm_logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # Flatten the tokens
+            loss = self.loss_function(
+                lm_logits,
+                labels,
+                vocab_size=self.config.vocab_size,
+                **kwargs,
+            )
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+            cross_attentions=transformer_outputs.cross_attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+        past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
+    ) -> Tuple[Tuple[torch.Tensor]]:
+        """
+        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+        [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+        beam_idx at every generation step.
+        """
+        return tuple(
+            tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
+            for layer_past in past_key_values
+        )
+
+
+
+from transformers import AutoConfig, AutoModel
+AutoConfig.register("parallel-gpt2", ParallelGPT2Config)
+AutoModel.register(ParallelGPT2Config, ParallelGPT2LMHeadModel)
+
+__all__ = [
+    "ParallelGPT2LMHeadModel",
+    "ParallelGPT2Model",
+    "ParallelGPT2Config",
+]
+
+
+if __name__  == "__main__":
+    cg = ParallelGPT2Config.from_pretrained("gpt2-medium", architectures=["ParallelGPT2LMHeadModel"])
+    model = ParallelGPT2LMHeadModel(cg)
+    from src.utils.model_utlis import print_trainable_parameters
+    print_trainable_parameters(model)
+    model(torch.randint(0, 10000, (1, 100)))
+    print()

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 4.998736842105263,
+    "total_flos": 1.0584067483285586e+18,
+    "train_loss": 3.584141966119902,
+    "train_runtime": 28526.973,
+    "train_samples_per_second": 19.98,
+    "train_steps_per_second": 0.624
+}

trainer_state.json

@@ -0,0 +1,1682 @@
+{
+  "best_metric": 3.101016044616699,
+  "best_model_checkpoint": "./output/models/parallel-gpt2-medium-wikitext/checkpoint-17500",
+  "epoch": 4.998736842105263,
+  "eval_steps": 500,
+  "global_step": 17810,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.028070175438596492,
+      "grad_norm": 2.028989553451538,
+      "learning_rate": 5.614823133071308e-06,
+      "loss": 9.2256,
+      "step": 100
+    },
+    {
+      "epoch": 0.056140350877192984,
+      "grad_norm": 1.8919556140899658,
+      "learning_rate": 1.1229646266142617e-05,
+      "loss": 7.8733,
+      "step": 200
+    },
+    {
+      "epoch": 0.08421052631578947,
+      "grad_norm": 1.433174967765808,
+      "learning_rate": 1.6844469399213926e-05,
+      "loss": 7.0853,
+      "step": 300
+    },
+    {
+      "epoch": 0.11228070175438597,
+      "grad_norm": 1.4522157907485962,
+      "learning_rate": 2.2459292532285233e-05,
+      "loss": 6.6702,
+      "step": 400
+    },
+    {
+      "epoch": 0.14035087719298245,
+      "grad_norm": 1.322840929031372,
+      "learning_rate": 2.8074115665356544e-05,
+      "loss": 6.4455,
+      "step": 500
+    },
+    {
+      "epoch": 0.14035087719298245,
+      "eval_accuracy": 0.17660504282313108,
+      "eval_bleu": 0.02573169351336114,
+      "eval_loss": 6.331261157989502,
+      "eval_perplexity": 561.8647474275982,
+      "eval_runtime": 20.1474,
+      "eval_samples_per_second": 56.633,
+      "eval_steps_per_second": 3.574,
+      "step": 500
+    },
+    {
+      "epoch": 0.16842105263157894,
+      "grad_norm": 1.5128698348999023,
+      "learning_rate": 3.368893879842785e-05,
+      "loss": 6.2694,
+      "step": 600
+    },
+    {
+      "epoch": 0.19649122807017544,
+      "grad_norm": 1.4778157472610474,
+      "learning_rate": 3.930376193149916e-05,
+      "loss": 6.1217,
+      "step": 700
+    },
+    {
+      "epoch": 0.22456140350877193,
+      "grad_norm": 1.1647950410842896,
+      "learning_rate": 4.4918585064570466e-05,
+      "loss": 5.9897,
+      "step": 800
+    },
+    {
+      "epoch": 0.25263157894736843,
+      "grad_norm": 1.3264423608779907,
+      "learning_rate": 5.053340819764178e-05,
+      "loss": 5.8659,
+      "step": 900
+    },
+    {
+      "epoch": 0.2807017543859649,
+      "grad_norm": 1.521269679069519,
+      "learning_rate": 5.614823133071309e-05,
+      "loss": 5.7254,
+      "step": 1000
+    },
+    {
+      "epoch": 0.2807017543859649,
+      "eval_accuracy": 0.2136290386834618,
+      "eval_bleu": 0.045427591918014404,
+      "eval_loss": 5.623491287231445,
+      "eval_perplexity": 276.85427569402634,
+      "eval_runtime": 20.1737,
+      "eval_samples_per_second": 56.559,
+      "eval_steps_per_second": 3.569,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3087719298245614,
+      "grad_norm": 1.1612671613693237,
+      "learning_rate": 6.176305446378439e-05,
+      "loss": 5.5759,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3368421052631579,
+      "grad_norm": 1.1074801683425903,
+      "learning_rate": 6.73778775968557e-05,
+      "loss": 5.4609,
+      "step": 1200
+    },
+    {
+      "epoch": 0.3649122807017544,
+      "grad_norm": 1.0632935762405396,
+      "learning_rate": 7.299270072992701e-05,
+      "loss": 5.3257,
+      "step": 1300
+    },
+    {
+      "epoch": 0.3929824561403509,
+      "grad_norm": 1.1046158075332642,
+      "learning_rate": 7.860752386299832e-05,
+      "loss": 5.1946,
+      "step": 1400
+    },
+    {
+      "epoch": 0.42105263157894735,
+      "grad_norm": 1.1495596170425415,
+      "learning_rate": 8.422234699606962e-05,
+      "loss": 5.1084,
+      "step": 1500
+    },
+    {
+      "epoch": 0.42105263157894735,
+      "eval_accuracy": 0.2575530545053601,
+      "eval_bleu": 0.06490299322624599,
+      "eval_loss": 4.982165813446045,
+      "eval_perplexity": 145.78979346372705,
+      "eval_runtime": 20.1855,
+      "eval_samples_per_second": 56.526,
+      "eval_steps_per_second": 3.567,
+      "step": 1500
+    },
+    {
+      "epoch": 0.44912280701754387,
+      "grad_norm": 0.9580332040786743,
+      "learning_rate": 8.983717012914093e-05,
+      "loss": 4.9932,
+      "step": 1600
+    },
+    {
+      "epoch": 0.47719298245614034,
+      "grad_norm": 0.9515564441680908,
+      "learning_rate": 9.545199326221224e-05,
+      "loss": 4.8819,
+      "step": 1700
+    },
+    {
+      "epoch": 0.5052631578947369,
+      "grad_norm": 0.9855053424835205,
+      "learning_rate": 9.98814648449685e-05,
+      "loss": 4.7868,
+      "step": 1800
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0349925756454468,
+      "learning_rate": 9.925759560796058e-05,
+      "loss": 4.6832,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5614035087719298,
+      "grad_norm": 1.0200501680374146,
+      "learning_rate": 9.863372637095265e-05,
+      "loss": 4.5994,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5614035087719298,
+      "eval_accuracy": 0.29288759924026103,
+      "eval_bleu": 0.07406906637840037,
+      "eval_loss": 4.5052409172058105,
+      "eval_perplexity": 90.49014205518364,
+      "eval_runtime": 19.5875,
+      "eval_samples_per_second": 58.251,
+      "eval_steps_per_second": 3.676,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5894736842105263,
+      "grad_norm": 0.9353361129760742,
+      "learning_rate": 9.800985713394473e-05,
+      "loss": 4.5176,
+      "step": 2100
+    },
+    {
+      "epoch": 0.6175438596491228,
+      "grad_norm": 0.9589385390281677,
+      "learning_rate": 9.73859878969368e-05,
+      "loss": 4.4355,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6456140350877193,
+      "grad_norm": 1.0331965684890747,
+      "learning_rate": 9.676211865992888e-05,
+      "loss": 4.3647,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6736842105263158,
+      "grad_norm": 0.9204744100570679,
+      "learning_rate": 9.613824942292095e-05,
+      "loss": 4.2935,
+      "step": 2400
+    },
+    {
+      "epoch": 0.7017543859649122,
+      "grad_norm": 0.9139348268508911,
+      "learning_rate": 9.551438018591303e-05,
+      "loss": 4.2338,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7017543859649122,
+      "eval_accuracy": 0.32728917628977,
+      "eval_bleu": 0.09369678160356684,
+      "eval_loss": 4.137840747833252,
+      "eval_perplexity": 62.66736062058415,
+      "eval_runtime": 20.1595,
+      "eval_samples_per_second": 56.598,
+      "eval_steps_per_second": 3.572,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7298245614035088,
+      "grad_norm": 0.9106621146202087,
+      "learning_rate": 9.489051094890511e-05,
+      "loss": 4.1694,
+      "step": 2600
+    },
+    {
+      "epoch": 0.7578947368421053,
+      "grad_norm": 0.9036598801612854,
+      "learning_rate": 9.426664171189718e-05,
+      "loss": 4.1386,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7859649122807018,
+      "grad_norm": 0.8367689847946167,
+      "learning_rate": 9.364277247488927e-05,
+      "loss": 4.0731,
+      "step": 2800
+    },
+    {
+      "epoch": 0.8140350877192982,
+      "grad_norm": 0.9330971837043762,
+      "learning_rate": 9.301890323788135e-05,
+      "loss": 4.0342,
+      "step": 2900
+    },
+    {
+      "epoch": 0.8421052631578947,
+      "grad_norm": 0.847179114818573,
+      "learning_rate": 9.239503400087343e-05,
+      "loss": 3.9975,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8421052631578947,
+      "eval_accuracy": 0.3465208187155545,
+      "eval_bleu": 0.10305263495616372,
+      "eval_loss": 3.928612470626831,
+      "eval_perplexity": 50.83639172374704,
+      "eval_runtime": 20.1917,
+      "eval_samples_per_second": 56.508,
+      "eval_steps_per_second": 3.566,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8701754385964913,
+      "grad_norm": 0.8944078087806702,
+      "learning_rate": 9.17711647638655e-05,
+      "loss": 3.9805,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8982456140350877,
+      "grad_norm": 0.8461546301841736,
+      "learning_rate": 9.114729552685758e-05,
+      "loss": 3.9468,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9263157894736842,
+      "grad_norm": 0.8034185767173767,
+      "learning_rate": 9.052342628984965e-05,
+      "loss": 3.9049,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9543859649122807,
+      "grad_norm": 0.8450652360916138,
+      "learning_rate": 8.989955705284174e-05,
+      "loss": 3.8727,
+      "step": 3400
+    },
+    {
+      "epoch": 0.9824561403508771,
+      "grad_norm": 0.7972449064254761,
+      "learning_rate": 8.92756878158338e-05,
+      "loss": 3.8648,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9824561403508771,
+      "eval_accuracy": 0.35825016727450926,
+      "eval_bleu": 0.11656022916295831,
+      "eval_loss": 3.7925570011138916,
+      "eval_perplexity": 44.36970874398872,
+      "eval_runtime": 20.1635,
+      "eval_samples_per_second": 56.587,
+      "eval_steps_per_second": 3.571,
+      "step": 3500
+    },
+    {
+      "epoch": 1.0103859649122806,
+      "grad_norm": 0.766300618648529,
+      "learning_rate": 8.865181857882589e-05,
+      "loss": 3.8193,
+      "step": 3600
+    },
+    {
+      "epoch": 1.0384561403508772,
+      "grad_norm": 0.8034301996231079,
+      "learning_rate": 8.802794934181796e-05,
+      "loss": 3.7801,
+      "step": 3700
+    },
+    {
+      "epoch": 1.0665263157894738,
+      "grad_norm": 0.8166210651397705,
+      "learning_rate": 8.740408010481004e-05,
+      "loss": 3.7505,
+      "step": 3800
+    },
+    {
+      "epoch": 1.0945964912280701,
+      "grad_norm": 0.7962071895599365,
+      "learning_rate": 8.678021086780212e-05,
+      "loss": 3.7398,
+      "step": 3900
+    },
+    {
+      "epoch": 1.1226666666666667,
+      "grad_norm": 0.7868255972862244,
+      "learning_rate": 8.615634163079419e-05,
+      "loss": 3.7164,
+      "step": 4000
+    },
+    {
+      "epoch": 1.1226666666666667,
+      "eval_accuracy": 0.3667368320049895,
+      "eval_bleu": 0.1225879386890452,
+      "eval_loss": 3.6986546516418457,
+      "eval_perplexity": 40.39292523319135,
+      "eval_runtime": 20.1969,
+      "eval_samples_per_second": 56.494,
+      "eval_steps_per_second": 3.565,
+      "step": 4000
+    },
+    {
+      "epoch": 1.150736842105263,
+      "grad_norm": 0.7974035739898682,
+      "learning_rate": 8.553247239378627e-05,
+      "loss": 3.7099,
+      "step": 4100
+    },
+    {
+      "epoch": 1.1788070175438596,
+      "grad_norm": 0.7970170974731445,
+      "learning_rate": 8.490860315677834e-05,
+      "loss": 3.6871,
+      "step": 4200
+    },
+    {
+      "epoch": 1.2068771929824562,
+      "grad_norm": 0.7822019457817078,
+      "learning_rate": 8.428473391977042e-05,
+      "loss": 3.6628,
+      "step": 4300
+    },
+    {
+      "epoch": 1.2349473684210526,
+      "grad_norm": 0.7597511410713196,
+      "learning_rate": 8.36608646827625e-05,
+      "loss": 3.6631,
+      "step": 4400
+    },
+    {
+      "epoch": 1.2630175438596491,
+      "grad_norm": 0.7736372947692871,
+      "learning_rate": 8.303699544575458e-05,
+      "loss": 3.6639,
+      "step": 4500
+    },
+    {
+      "epoch": 1.2630175438596491,
+      "eval_accuracy": 0.37342095861787133,
+      "eval_bleu": 0.12824268044271067,
+      "eval_loss": 3.622089147567749,
+      "eval_perplexity": 37.415653048434656,
+      "eval_runtime": 20.1594,
+      "eval_samples_per_second": 56.599,
+      "eval_steps_per_second": 3.572,
+      "step": 4500
+    },
+    {
+      "epoch": 1.2910877192982455,
+      "grad_norm": 0.7472370266914368,
+      "learning_rate": 8.241312620874664e-05,
+      "loss": 3.6404,
+      "step": 4600
+    },
+    {
+      "epoch": 1.319157894736842,
+      "grad_norm": 0.771800696849823,
+      "learning_rate": 8.178925697173873e-05,
+      "loss": 3.6235,
+      "step": 4700
+    },
+    {
+      "epoch": 1.3472280701754387,
+      "grad_norm": 0.7624268531799316,
+      "learning_rate": 8.11653877347308e-05,
+      "loss": 3.6123,
+      "step": 4800
+    },
+    {
+      "epoch": 1.375298245614035,
+      "grad_norm": 0.7671138644218445,
+      "learning_rate": 8.054151849772288e-05,
+      "loss": 3.592,
+      "step": 4900
+    },
+    {
+      "epoch": 1.4033684210526316,
+      "grad_norm": 0.7753613591194153,
+      "learning_rate": 7.991764926071496e-05,
+      "loss": 3.582,
+      "step": 5000
+    },
+    {
+      "epoch": 1.4033684210526316,
+      "eval_accuracy": 0.3795687787375893,
+      "eval_bleu": 0.1276585909940598,
+      "eval_loss": 3.5575244426727295,
+      "eval_perplexity": 35.07625629269647,
+      "eval_runtime": 20.1451,
+      "eval_samples_per_second": 56.639,
+      "eval_steps_per_second": 3.574,
+      "step": 5000
+    },
+    {
+      "epoch": 1.431438596491228,
+      "grad_norm": 0.7279810309410095,
+      "learning_rate": 7.929378002370703e-05,
+      "loss": 3.5846,
+      "step": 5100
+    },
+    {
+      "epoch": 1.4595087719298245,
+      "grad_norm": 0.728085458278656,
+      "learning_rate": 7.866991078669911e-05,
+      "loss": 3.5677,
+      "step": 5200
+    },
+    {
+      "epoch": 1.4875789473684211,
+      "grad_norm": 0.7683869004249573,
+      "learning_rate": 7.804604154969118e-05,
+      "loss": 3.5452,
+      "step": 5300
+    },
+    {
+      "epoch": 1.5156491228070177,
+      "grad_norm": 0.7563459277153015,
+      "learning_rate": 7.742217231268327e-05,
+      "loss": 3.5557,
+      "step": 5400
+    },
+    {
+      "epoch": 1.543719298245614,
+      "grad_norm": 0.7477959394454956,
+      "learning_rate": 7.679830307567533e-05,
+      "loss": 3.5315,
+      "step": 5500
+    },
+    {
+      "epoch": 1.543719298245614,
+      "eval_accuracy": 0.38395261312340273,
+      "eval_bleu": 0.13124123866011556,
+      "eval_loss": 3.5063869953155518,
+      "eval_perplexity": 33.32763708661619,
+      "eval_runtime": 20.2862,
+      "eval_samples_per_second": 56.245,
+      "eval_steps_per_second": 3.549,
+      "step": 5500
+    },
+    {
+      "epoch": 1.5717894736842104,
+      "grad_norm": 0.732016921043396,
+      "learning_rate": 7.617443383866742e-05,
+      "loss": 3.5185,
+      "step": 5600
+    },
+    {
+      "epoch": 1.599859649122807,
+      "grad_norm": 0.7203788757324219,
+      "learning_rate": 7.555056460165949e-05,
+      "loss": 3.5216,
+      "step": 5700
+    },
+    {
+      "epoch": 1.6279298245614036,
+      "grad_norm": 0.7561742663383484,
+      "learning_rate": 7.492669536465158e-05,
+      "loss": 3.5057,
+      "step": 5800
+    },
+    {
+      "epoch": 1.6560000000000001,
+      "grad_norm": 0.7377160787582397,
+      "learning_rate": 7.430282612764365e-05,
+      "loss": 3.5058,
+      "step": 5900
+    },
+    {
+      "epoch": 1.6840701754385965,
+      "grad_norm": 0.8002681136131287,
+      "learning_rate": 7.367895689063573e-05,
+      "loss": 3.5025,
+      "step": 6000
+    },
+    {
+      "epoch": 1.6840701754385965,
+      "eval_accuracy": 0.3880588703466202,
+      "eval_bleu": 0.13662311841214886,
+      "eval_loss": 3.4594311714172363,
+      "eval_perplexity": 31.798883255478383,
+      "eval_runtime": 20.2931,
+      "eval_samples_per_second": 56.226,
+      "eval_steps_per_second": 3.548,
+      "step": 6000
+    },
+    {
+      "epoch": 1.7121403508771929,
+      "grad_norm": 0.7451775074005127,
+      "learning_rate": 7.30550876536278e-05,
+      "loss": 3.4714,
+      "step": 6100
+    },
+    {
+      "epoch": 1.7402105263157894,
+      "grad_norm": 0.7121933102607727,
+      "learning_rate": 7.243121841661989e-05,
+      "loss": 3.474,
+      "step": 6200
+    },
+    {
+      "epoch": 1.768280701754386,
+      "grad_norm": 0.7507015466690063,
+      "learning_rate": 7.180734917961195e-05,
+      "loss": 3.4577,
+      "step": 6300
+    },
+    {
+      "epoch": 1.7963508771929826,
+      "grad_norm": 0.7061077356338501,
+      "learning_rate": 7.118347994260404e-05,
+      "loss": 3.463,
+      "step": 6400
+    },
+    {
+      "epoch": 1.824421052631579,
+      "grad_norm": 0.7301498055458069,
+      "learning_rate": 7.055961070559612e-05,
+      "loss": 3.4462,
+      "step": 6500
+    },
+    {
+      "epoch": 1.824421052631579,
+      "eval_accuracy": 0.3919063982392698,
+      "eval_bleu": 0.13096345408557636,
+      "eval_loss": 3.420844554901123,
+      "eval_perplexity": 30.5952434755694,
+      "eval_runtime": 20.2154,
+      "eval_samples_per_second": 56.442,
+      "eval_steps_per_second": 3.562,
+      "step": 6500
+    },
+    {
+      "epoch": 1.8524912280701753,
+      "grad_norm": 0.713631272315979,
+      "learning_rate": 6.993574146858819e-05,
+      "loss": 3.4245,
+      "step": 6600
+    },
+    {
+      "epoch": 1.8805614035087719,
+      "grad_norm": 0.7331680059432983,
+      "learning_rate": 6.931187223158027e-05,
+      "loss": 3.4289,
+      "step": 6700
+    },
+    {
+      "epoch": 1.9086315789473685,
+      "grad_norm": 0.7402526140213013,
+      "learning_rate": 6.868800299457234e-05,
+      "loss": 3.434,
+      "step": 6800
+    },
+    {
+      "epoch": 1.936701754385965,
+      "grad_norm": 0.7350389361381531,
+      "learning_rate": 6.806413375756442e-05,
+      "loss": 3.413,
+      "step": 6900
+    },
+    {
+      "epoch": 1.9647719298245614,
+      "grad_norm": 0.7257172465324402,
+      "learning_rate": 6.744026452055649e-05,
+      "loss": 3.4167,
+      "step": 7000
+    },
+    {
+      "epoch": 1.9647719298245614,
+      "eval_accuracy": 0.39561171067207085,
+      "eval_bleu": 0.1355256129038967,
+      "eval_loss": 3.3863015174865723,
+      "eval_perplexity": 29.556435911658944,
+      "eval_runtime": 20.2056,
+      "eval_samples_per_second": 56.469,
+      "eval_steps_per_second": 3.563,
+      "step": 7000
+    },
+    {
+      "epoch": 1.9928421052631577,
+      "grad_norm": 0.7187970876693726,
+      "learning_rate": 6.681639528354857e-05,
+      "loss": 3.4086,
+      "step": 7100
+    },
+    {
+      "epoch": 2.020771929824561,
+      "grad_norm": 0.7202402353286743,
+      "learning_rate": 6.619252604654064e-05,
+      "loss": 3.3372,
+      "step": 7200
+    },
+    {
+      "epoch": 2.048842105263158,
+      "grad_norm": 0.7403327822685242,
+      "learning_rate": 6.556865680953273e-05,
+      "loss": 3.3079,
+      "step": 7300
+    },
+    {
+      "epoch": 2.0769122807017544,
+      "grad_norm": 0.7321439385414124,
+      "learning_rate": 6.49447875725248e-05,
+      "loss": 3.3128,
+      "step": 7400
+    },
+    {
+      "epoch": 2.104982456140351,
+      "grad_norm": 0.7240020632743835,
+      "learning_rate": 6.432091833551688e-05,
+      "loss": 3.2967,
+      "step": 7500
+    },
+    {
+      "epoch": 2.104982456140351,
+      "eval_accuracy": 0.39890408423952856,
+      "eval_bleu": 0.1316612565684532,
+      "eval_loss": 3.3547873497009277,
+      "eval_perplexity": 28.639513342041703,
+      "eval_runtime": 20.2147,
+      "eval_samples_per_second": 56.444,
+      "eval_steps_per_second": 3.562,
+      "step": 7500
+    },
+    {
+      "epoch": 2.1330526315789475,
+      "grad_norm": 0.7483673095703125,
+      "learning_rate": 6.369704909850896e-05,
+      "loss": 3.2857,
+      "step": 7600
+    },
+    {
+      "epoch": 2.1611228070175437,
+      "grad_norm": 0.7482850551605225,
+      "learning_rate": 6.307317986150103e-05,
+      "loss": 3.3038,
+      "step": 7700
+    },
+    {
+      "epoch": 2.1891929824561402,
+      "grad_norm": 0.7295832633972168,
+      "learning_rate": 6.244931062449311e-05,
+      "loss": 3.284,
+      "step": 7800
+    },
+    {
+      "epoch": 2.217263157894737,
+      "grad_norm": 0.7610414028167725,
+      "learning_rate": 6.182544138748518e-05,
+      "loss": 3.2862,
+      "step": 7900
+    },
+    {
+      "epoch": 2.2453333333333334,
+      "grad_norm": 0.7399555444717407,
+      "learning_rate": 6.120157215047726e-05,
+      "loss": 3.2909,
+      "step": 8000
+    },
+    {
+      "epoch": 2.2453333333333334,
+      "eval_accuracy": 0.4014682461149906,
+      "eval_bleu": 0.13811168690281525,
+      "eval_loss": 3.329040050506592,
+      "eval_perplexity": 27.911535174787044,
+      "eval_runtime": 20.1568,
+      "eval_samples_per_second": 56.606,
+      "eval_steps_per_second": 3.572,
+      "step": 8000
+    },
+    {
+      "epoch": 2.27340350877193,
+      "grad_norm": 0.7306208610534668,
+      "learning_rate": 6.057770291346934e-05,
+      "loss": 3.2853,
+      "step": 8100
+    },
+    {
+      "epoch": 2.301473684210526,
+      "grad_norm": 0.7453253269195557,
+      "learning_rate": 5.9953833676461415e-05,
+      "loss": 3.2744,
+      "step": 8200
+    },
+    {
+      "epoch": 2.3295438596491227,
+      "grad_norm": 0.7369946837425232,
+      "learning_rate": 5.932996443945349e-05,
+      "loss": 3.2604,
+      "step": 8300
+    },
+    {
+      "epoch": 2.3576140350877193,
+      "grad_norm": 0.7455064654350281,
+      "learning_rate": 5.870609520244557e-05,
+      "loss": 3.2622,
+      "step": 8400
+    },
+    {
+      "epoch": 2.385684210526316,
+      "grad_norm": 0.7593878507614136,
+      "learning_rate": 5.808222596543764e-05,
+      "loss": 3.2593,
+      "step": 8500
+    },
+    {
+      "epoch": 2.385684210526316,
+      "eval_accuracy": 0.40385506702546087,
+      "eval_bleu": 0.14220030200690656,
+      "eval_loss": 3.3044025897979736,
+      "eval_perplexity": 27.232267893585384,
+      "eval_runtime": 20.1746,
+      "eval_samples_per_second": 56.556,
+      "eval_steps_per_second": 3.569,
+      "step": 8500
+    },
+    {
+      "epoch": 2.4137543859649124,
+      "grad_norm": 0.7376691102981567,
+      "learning_rate": 5.745835672842972e-05,
+      "loss": 3.2582,
+      "step": 8600
+    },
+    {
+      "epoch": 2.441824561403509,
+      "grad_norm": 0.7452288269996643,
+      "learning_rate": 5.683448749142181e-05,
+      "loss": 3.2491,
+      "step": 8700
+    },
+    {
+      "epoch": 2.469894736842105,
+      "grad_norm": 0.749320924282074,
+      "learning_rate": 5.6210618254413884e-05,
+      "loss": 3.24,
+      "step": 8800
+    },
+    {
+      "epoch": 2.4979649122807017,
+      "grad_norm": 0.7482838034629822,
+      "learning_rate": 5.558674901740596e-05,
+      "loss": 3.2374,
+      "step": 8900
+    },
+    {
+      "epoch": 2.5260350877192983,
+      "grad_norm": 0.7401648163795471,
+      "learning_rate": 5.4962879780398035e-05,
+      "loss": 3.2408,
+      "step": 9000
+    },
+    {
+      "epoch": 2.5260350877192983,
+      "eval_accuracy": 0.4061330845419506,
+      "eval_bleu": 0.14119139857697727,
+      "eval_loss": 3.282592296600342,
+      "eval_perplexity": 26.644754356923094,
+      "eval_runtime": 20.1938,
+      "eval_samples_per_second": 56.503,
+      "eval_steps_per_second": 3.565,
+      "step": 9000
+    },
+    {
+      "epoch": 2.554105263157895,
+      "grad_norm": 0.7236805558204651,
+      "learning_rate": 5.433901054339011e-05,
+      "loss": 3.2434,
+      "step": 9100
+    },
+    {
+      "epoch": 2.582175438596491,
+      "grad_norm": 0.7386437058448792,
+      "learning_rate": 5.371514130638219e-05,
+      "loss": 3.2445,
+      "step": 9200
+    },
+    {
+      "epoch": 2.6102456140350876,
+      "grad_norm": 0.7296267747879028,
+      "learning_rate": 5.309127206937426e-05,
+      "loss": 3.2257,
+      "step": 9300
+    },
+    {
+      "epoch": 2.638315789473684,
+      "grad_norm": 0.7237492203712463,
+      "learning_rate": 5.246740283236634e-05,
+      "loss": 3.2336,
+      "step": 9400
+    },
+    {
+      "epoch": 2.6663859649122807,
+      "grad_norm": 0.7474483847618103,
+      "learning_rate": 5.1843533595358414e-05,
+      "loss": 3.2278,
+      "step": 9500
+    },
+    {
+      "epoch": 2.6663859649122807,
+      "eval_accuracy": 0.40903479395464354,
+      "eval_bleu": 0.14356756322892422,
+      "eval_loss": 3.2591938972473145,
+      "eval_perplexity": 26.028547000163957,
+      "eval_runtime": 20.1915,
+      "eval_samples_per_second": 56.509,
+      "eval_steps_per_second": 3.566,
+      "step": 9500
+    },
+    {
+      "epoch": 2.6944561403508773,
+      "grad_norm": 0.7365037798881531,
+      "learning_rate": 5.12196643583505e-05,
+      "loss": 3.2235,
+      "step": 9600
+    },
+    {
+      "epoch": 2.722526315789474,
+      "grad_norm": 0.7330195307731628,
+      "learning_rate": 5.059579512134257e-05,
+      "loss": 3.2179,
+      "step": 9700
+    },
+    {
+      "epoch": 2.75059649122807,
+      "grad_norm": 0.7449477314949036,
+      "learning_rate": 4.997192588433465e-05,
+      "loss": 3.2178,
+      "step": 9800
+    },
+    {
+      "epoch": 2.7786666666666666,
+      "grad_norm": 0.7243569493293762,
+      "learning_rate": 4.9348056647326725e-05,
+      "loss": 3.2126,
+      "step": 9900
+    },
+    {
+      "epoch": 2.806736842105263,
+      "grad_norm": 0.7603092789649963,
+      "learning_rate": 4.87241874103188e-05,
+      "loss": 3.2172,
+      "step": 10000
+    },
+    {
+      "epoch": 2.806736842105263,
+      "eval_accuracy": 0.4104792232637077,
+      "eval_bleu": 0.1411554473599086,
+      "eval_loss": 3.2415478229522705,
+      "eval_perplexity": 25.573274029997943,
+      "eval_runtime": 19.9283,
+      "eval_samples_per_second": 57.255,
+      "eval_steps_per_second": 3.613,
+      "step": 10000
+    },
+    {
+      "epoch": 2.8348070175438598,
+      "grad_norm": 0.7319552302360535,
+      "learning_rate": 4.8100318173310876e-05,
+      "loss": 3.2138,
+      "step": 10100
+    },
+    {
+      "epoch": 2.862877192982456,
+      "grad_norm": 0.7321156859397888,
+      "learning_rate": 4.747644893630295e-05,
+      "loss": 3.2114,
+      "step": 10200
+    },
+    {
+      "epoch": 2.8909473684210525,
+      "grad_norm": 0.7428148984909058,
+      "learning_rate": 4.685257969929503e-05,
+      "loss": 3.1985,
+      "step": 10300
+    },
+    {
+      "epoch": 2.919017543859649,
+      "grad_norm": 0.7498809099197388,
+      "learning_rate": 4.6228710462287104e-05,
+      "loss": 3.1861,
+      "step": 10400
+    },
+    {
+      "epoch": 2.9470877192982456,
+      "grad_norm": 0.7468050718307495,
+      "learning_rate": 4.560484122527918e-05,
+      "loss": 3.2145,
+      "step": 10500
+    },
+    {
+      "epoch": 2.9470877192982456,
+      "eval_accuracy": 0.41254477431006226,
+      "eval_bleu": 0.14022835224132563,
+      "eval_loss": 3.222651243209839,
+      "eval_perplexity": 25.094563855273414,
+      "eval_runtime": 20.1985,
+      "eval_samples_per_second": 56.489,
+      "eval_steps_per_second": 3.565,
+      "step": 10500
+    },
+    {
+      "epoch": 2.9751578947368422,
+      "grad_norm": 0.7379462122917175,
+      "learning_rate": 4.498097198827126e-05,
+      "loss": 3.1799,
+      "step": 10600
+    },
+    {
+      "epoch": 3.0030877192982457,
+      "grad_norm": 0.7374199032783508,
+      "learning_rate": 4.435710275126334e-05,
+      "loss": 3.1681,
+      "step": 10700
+    },
+    {
+      "epoch": 3.0311578947368423,
+      "grad_norm": 0.7500795722007751,
+      "learning_rate": 4.3733233514255414e-05,
+      "loss": 3.096,
+      "step": 10800
+    },
+    {
+      "epoch": 3.0592280701754384,
+      "grad_norm": 0.7644433975219727,
+      "learning_rate": 4.3109364277247496e-05,
+      "loss": 3.0842,
+      "step": 10900
+    },
+    {
+      "epoch": 3.087298245614035,
+      "grad_norm": 0.7695332169532776,
+      "learning_rate": 4.248549504023957e-05,
+      "loss": 3.0749,
+      "step": 11000
+    },
+    {
+      "epoch": 3.087298245614035,
+      "eval_accuracy": 0.41432503771708207,
+      "eval_bleu": 0.1413248900872156,
+      "eval_loss": 3.209906578063965,
+      "eval_perplexity": 24.776771422507743,
+      "eval_runtime": 20.1941,
+      "eval_samples_per_second": 56.502,
+      "eval_steps_per_second": 3.565,
+      "step": 11000
+    },
+    {
+      "epoch": 3.1153684210526316,
+      "grad_norm": 0.7983810901641846,
+      "learning_rate": 4.186162580323165e-05,
+      "loss": 3.0886,
+      "step": 11100
+    },
+    {
+      "epoch": 3.143438596491228,
+      "grad_norm": 0.7757524847984314,
+      "learning_rate": 4.1237756566223724e-05,
+      "loss": 3.0873,
+      "step": 11200
+    },
+    {
+      "epoch": 3.1715087719298247,
+      "grad_norm": 0.8084014654159546,
+      "learning_rate": 4.06138873292158e-05,
+      "loss": 3.0882,
+      "step": 11300
+    },
+    {
+      "epoch": 3.199578947368421,
+      "grad_norm": 0.7784711718559265,
+      "learning_rate": 3.9990018092207875e-05,
+      "loss": 3.0807,
+      "step": 11400
+    },
+    {
+      "epoch": 3.2276491228070174,
+      "grad_norm": 0.7808659076690674,
+      "learning_rate": 3.936614885519995e-05,
+      "loss": 3.0777,
+      "step": 11500
+    },
+    {
+      "epoch": 3.2276491228070174,
+      "eval_accuracy": 0.4159836469355567,
+      "eval_bleu": 0.14196972146127593,
+      "eval_loss": 3.1977927684783936,
+      "eval_perplexity": 24.478440938871064,
+      "eval_runtime": 20.2549,
+      "eval_samples_per_second": 56.332,
+      "eval_steps_per_second": 3.555,
+      "step": 11500
+    },
+    {
+      "epoch": 3.255719298245614,
+      "grad_norm": 0.7880111336708069,
+      "learning_rate": 3.874227961819203e-05,
+      "loss": 3.0907,
+      "step": 11600
+    },
+    {
+      "epoch": 3.2837894736842106,
+      "grad_norm": 0.7764289975166321,
+      "learning_rate": 3.81184103811841e-05,
+      "loss": 3.0829,
+      "step": 11700
+    },
+    {
+      "epoch": 3.311859649122807,
+      "grad_norm": 0.7774072885513306,
+      "learning_rate": 3.749454114417618e-05,
+      "loss": 3.084,
+      "step": 11800
+    },
+    {
+      "epoch": 3.3399298245614037,
+      "grad_norm": 0.7633748650550842,
+      "learning_rate": 3.687067190716826e-05,
+      "loss": 3.0832,
+      "step": 11900
+    },
+    {
+      "epoch": 3.368,
+      "grad_norm": 0.7821294069290161,
+      "learning_rate": 3.624680267016034e-05,
+      "loss": 3.0743,
+      "step": 12000
+    },
+    {
+      "epoch": 3.368,
+      "eval_accuracy": 0.41744863751592426,
+      "eval_bleu": 0.1437616042304935,
+      "eval_loss": 3.185511589050293,
+      "eval_perplexity": 24.1796552890801,
+      "eval_runtime": 20.1722,
+      "eval_samples_per_second": 56.563,
+      "eval_steps_per_second": 3.569,
+      "step": 12000
+    },
+    {
+      "epoch": 3.3960701754385965,
+      "grad_norm": 0.7699295878410339,
+      "learning_rate": 3.562293343315241e-05,
+      "loss": 3.0603,
+      "step": 12100
+    },
+    {
+      "epoch": 3.424140350877193,
+      "grad_norm": 0.7893390655517578,
+      "learning_rate": 3.499906419614449e-05,
+      "loss": 3.064,
+      "step": 12200
+    },
+    {
+      "epoch": 3.4522105263157896,
+      "grad_norm": 0.7758604884147644,
+      "learning_rate": 3.4375194959136565e-05,
+      "loss": 3.0791,
+      "step": 12300
+    },
+    {
+      "epoch": 3.4802807017543858,
+      "grad_norm": 0.7830843925476074,
+      "learning_rate": 3.375132572212865e-05,
+      "loss": 3.0691,
+      "step": 12400
+    },
+    {
+      "epoch": 3.5083508771929823,
+      "grad_norm": 0.7860715389251709,
+      "learning_rate": 3.312745648512072e-05,
+      "loss": 3.0679,
+      "step": 12500
+    },
+    {
+      "epoch": 3.5083508771929823,
+      "eval_accuracy": 0.4182539539753076,
+      "eval_bleu": 0.13973142885022957,
+      "eval_loss": 3.1735119819641113,
+      "eval_perplexity": 23.891242805090002,
+      "eval_runtime": 20.2985,
+      "eval_samples_per_second": 56.211,
+      "eval_steps_per_second": 3.547,
+      "step": 12500
+    },
+    {
+      "epoch": 3.536421052631579,
+      "grad_norm": 0.7781902551651001,
+      "learning_rate": 3.25035872481128e-05,
+      "loss": 3.0577,
+      "step": 12600
+    },
+    {
+      "epoch": 3.5644912280701755,
+      "grad_norm": 0.7784900665283203,
+      "learning_rate": 3.1879718011104875e-05,
+      "loss": 3.0775,
+      "step": 12700
+    },
+    {
+      "epoch": 3.592561403508772,
+      "grad_norm": 0.8009820580482483,
+      "learning_rate": 3.125584877409695e-05,
+      "loss": 3.0532,
+      "step": 12800
+    },
+    {
+      "epoch": 3.6206315789473686,
+      "grad_norm": 0.7675787210464478,
+      "learning_rate": 3.0631979537089026e-05,
+      "loss": 3.0597,
+      "step": 12900
+    },
+    {
+      "epoch": 3.648701754385965,
+      "grad_norm": 0.7848743796348572,
+      "learning_rate": 3.0008110300081106e-05,
+      "loss": 3.0635,
+      "step": 13000
+    },
+    {
+      "epoch": 3.648701754385965,
+      "eval_accuracy": 0.4199536857363891,
+      "eval_bleu": 0.14234657853553717,
+      "eval_loss": 3.1599371433258057,
+      "eval_perplexity": 23.56911440636153,
+      "eval_runtime": 20.1704,
+      "eval_samples_per_second": 56.568,
+      "eval_steps_per_second": 3.57,
+      "step": 13000
+    },
+    {
+      "epoch": 3.6767719298245614,
+      "grad_norm": 0.7748121023178101,
+      "learning_rate": 2.938424106307318e-05,
+      "loss": 3.0489,
+      "step": 13100
+    },
+    {
+      "epoch": 3.704842105263158,
+      "grad_norm": 0.7777696251869202,
+      "learning_rate": 2.8760371826065257e-05,
+      "loss": 3.0603,
+      "step": 13200
+    },
+    {
+      "epoch": 3.7329122807017545,
+      "grad_norm": 0.7850595116615295,
+      "learning_rate": 2.8136502589057333e-05,
+      "loss": 3.0361,
+      "step": 13300
+    },
+    {
+      "epoch": 3.7609824561403506,
+      "grad_norm": 0.7980267405509949,
+      "learning_rate": 2.7512633352049412e-05,
+      "loss": 3.0565,
+      "step": 13400
+    },
+    {
+      "epoch": 3.7890526315789472,
+      "grad_norm": 0.8132106065750122,
+      "learning_rate": 2.6888764115041488e-05,
+      "loss": 3.0262,
+      "step": 13500
+    },
+    {
+      "epoch": 3.7890526315789472,
+      "eval_accuracy": 0.4210717048635117,
+      "eval_bleu": 0.14320922241168227,
+      "eval_loss": 3.148859977722168,
+      "eval_perplexity": 23.309476106050013,
+      "eval_runtime": 20.2248,
+      "eval_samples_per_second": 56.416,
+      "eval_steps_per_second": 3.56,
+      "step": 13500
+    },
+    {
+      "epoch": 3.817122807017544,
+      "grad_norm": 0.8002068400382996,
+      "learning_rate": 2.6264894878033564e-05,
+      "loss": 3.0504,
+      "step": 13600
+    },
+    {
+      "epoch": 3.8451929824561404,
+      "grad_norm": 0.7945306301116943,
+      "learning_rate": 2.564102564102564e-05,
+      "loss": 3.0412,
+      "step": 13700
+    },
+    {
+      "epoch": 3.873263157894737,
+      "grad_norm": 0.7735077738761902,
+      "learning_rate": 2.5017156404017715e-05,
+      "loss": 3.0193,
+      "step": 13800
+    },
+    {
+      "epoch": 3.9013333333333335,
+      "grad_norm": 0.7823268175125122,
+      "learning_rate": 2.4393287167009795e-05,
+      "loss": 3.0408,
+      "step": 13900
+    },
+    {
+      "epoch": 3.9294035087719297,
+      "grad_norm": 0.7943819165229797,
+      "learning_rate": 2.376941793000187e-05,
+      "loss": 3.0382,
+      "step": 14000
+    },
+    {
+      "epoch": 3.9294035087719297,
+      "eval_accuracy": 0.4223199453755559,
+      "eval_bleu": 0.1460635167412373,
+      "eval_loss": 3.139704465866089,
+      "eval_perplexity": 23.097039886291473,
+      "eval_runtime": 20.2485,
+      "eval_samples_per_second": 56.35,
+      "eval_steps_per_second": 3.556,
+      "step": 14000
+    },
+    {
+      "epoch": 3.9574736842105263,
+      "grad_norm": 0.7909451723098755,
+      "learning_rate": 2.314554869299395e-05,
+      "loss": 3.032,
+      "step": 14100
+    },
+    {
+      "epoch": 3.985543859649123,
+      "grad_norm": 0.7830550670623779,
+      "learning_rate": 2.2521679455986026e-05,
+      "loss": 3.0383,
+      "step": 14200
+    },
+    {
+      "epoch": 4.013473684210527,
+      "grad_norm": 0.7939472794532776,
+      "learning_rate": 2.1897810218978105e-05,
+      "loss": 2.9744,
+      "step": 14300
+    },
+    {
+      "epoch": 4.041543859649122,
+      "grad_norm": 0.8031191229820251,
+      "learning_rate": 2.127394098197018e-05,
+      "loss": 2.9426,
+      "step": 14400
+    },
+    {
+      "epoch": 4.069614035087719,
+      "grad_norm": 0.8071146607398987,
+      "learning_rate": 2.0650071744962256e-05,
+      "loss": 2.9525,
+      "step": 14500
+    },
+    {
+      "epoch": 4.069614035087719,
+      "eval_accuracy": 0.4232880385660912,
+      "eval_bleu": 0.1457336973656826,
+      "eval_loss": 3.1334879398345947,
+      "eval_perplexity": 22.95390190803781,
+      "eval_runtime": 20.1672,
+      "eval_samples_per_second": 56.577,
+      "eval_steps_per_second": 3.57,
+      "step": 14500
+    },
+    {
+      "epoch": 4.097684210526316,
+      "grad_norm": 0.8192425966262817,
+      "learning_rate": 2.0026202507954332e-05,
+      "loss": 2.9449,
+      "step": 14600
+    },
+    {
+      "epoch": 4.125754385964912,
+      "grad_norm": 0.8179590702056885,
+      "learning_rate": 1.940233327094641e-05,
+      "loss": 2.9534,
+      "step": 14700
+    },
+    {
+      "epoch": 4.153824561403509,
+      "grad_norm": 0.8264966607093811,
+      "learning_rate": 1.8778464033938487e-05,
+      "loss": 2.957,
+      "step": 14800
+    },
+    {
+      "epoch": 4.181894736842105,
+      "grad_norm": 0.8411971926689148,
+      "learning_rate": 1.8154594796930563e-05,
+      "loss": 2.9503,
+      "step": 14900
+    },
+    {
+      "epoch": 4.209964912280702,
+      "grad_norm": 0.818305253982544,
+      "learning_rate": 1.7530725559922642e-05,
+      "loss": 2.9621,
+      "step": 15000
+    },
+    {
+      "epoch": 4.209964912280702,
+      "eval_accuracy": 0.4239245812568591,
+      "eval_bleu": 0.14536464175084168,
+      "eval_loss": 3.1270124912261963,
+      "eval_perplexity": 22.805745303809648,
+      "eval_runtime": 12.3884,
+      "eval_samples_per_second": 92.102,
+      "eval_steps_per_second": 5.812,
+      "step": 15000
+    },
+    {
+      "epoch": 4.2380350877192985,
+      "grad_norm": 0.8167079091072083,
+      "learning_rate": 1.6906856322914718e-05,
+      "loss": 2.944,
+      "step": 15100
+    },
+    {
+      "epoch": 4.266105263157895,
+      "grad_norm": 0.837181031703949,
+      "learning_rate": 1.6282987085906794e-05,
+      "loss": 2.9407,
+      "step": 15200
+    },
+    {
+      "epoch": 4.294175438596492,
+      "grad_norm": 0.8356810808181763,
+      "learning_rate": 1.5659117848898873e-05,
+      "loss": 2.9364,
+      "step": 15300
+    },
+    {
+      "epoch": 4.322245614035087,
+      "grad_norm": 0.8358649015426636,
+      "learning_rate": 1.5035248611890947e-05,
+      "loss": 2.9307,
+      "step": 15400
+    },
+    {
+      "epoch": 4.350315789473684,
+      "grad_norm": 0.842452347278595,
+      "learning_rate": 1.4411379374883025e-05,
+      "loss": 2.9422,
+      "step": 15500
+    },
+    {
+      "epoch": 4.350315789473684,
+      "eval_accuracy": 0.42498777032717266,
+      "eval_bleu": 0.1467649753048137,
+      "eval_loss": 3.1211211681365967,
+      "eval_perplexity": 22.671784281214766,
+      "eval_runtime": 20.2024,
+      "eval_samples_per_second": 56.479,
+      "eval_steps_per_second": 3.564,
+      "step": 15500
+    },
+    {
+      "epoch": 4.3783859649122805,
+      "grad_norm": 0.8464500904083252,
+      "learning_rate": 1.37875101378751e-05,
+      "loss": 2.9435,
+      "step": 15600
+    },
+    {
+      "epoch": 4.406456140350877,
+      "grad_norm": 0.8314975500106812,
+      "learning_rate": 1.316364090086718e-05,
+      "loss": 2.9405,
+      "step": 15700
+    },
+    {
+      "epoch": 4.434526315789474,
+      "grad_norm": 0.8358827829360962,
+      "learning_rate": 1.2539771663859257e-05,
+      "loss": 2.9438,
+      "step": 15800
+    },
+    {
+      "epoch": 4.46259649122807,
+      "grad_norm": 0.8394728899002075,
+      "learning_rate": 1.1915902426851333e-05,
+      "loss": 2.9233,
+      "step": 15900
+    },
+    {
+      "epoch": 4.490666666666667,
+      "grad_norm": 0.839545488357544,
+      "learning_rate": 1.1292033189843409e-05,
+      "loss": 2.9224,
+      "step": 16000
+    },
+    {
+      "epoch": 4.490666666666667,
+      "eval_accuracy": 0.4256748594765614,
+      "eval_bleu": 0.1454115201862865,
+      "eval_loss": 3.1149473190307617,
+      "eval_perplexity": 22.53224330181115,
+      "eval_runtime": 20.2018,
+      "eval_samples_per_second": 56.48,
+      "eval_steps_per_second": 3.564,
+      "step": 16000
+    },
+    {
+      "epoch": 4.518736842105263,
+      "grad_norm": 0.847416341304779,
+      "learning_rate": 1.0668163952835486e-05,
+      "loss": 2.9372,
+      "step": 16100
+    },
+    {
+      "epoch": 4.54680701754386,
+      "grad_norm": 0.8180661201477051,
+      "learning_rate": 1.0044294715827562e-05,
+      "loss": 2.9422,
+      "step": 16200
+    },
+    {
+      "epoch": 4.574877192982456,
+      "grad_norm": 0.8318551182746887,
+      "learning_rate": 9.42042547881964e-06,
+      "loss": 2.9373,
+      "step": 16300
+    },
+    {
+      "epoch": 4.602947368421052,
+      "grad_norm": 0.8445524573326111,
+      "learning_rate": 8.796556241811717e-06,
+      "loss": 2.9212,
+      "step": 16400
+    },
+    {
+      "epoch": 4.631017543859649,
+      "grad_norm": 0.8220964670181274,
+      "learning_rate": 8.172687004803793e-06,
+      "loss": 2.9475,
+      "step": 16500
+    },
+    {
+      "epoch": 4.631017543859649,
+      "eval_accuracy": 0.4264193488416722,
+      "eval_bleu": 0.149677834207122,
+      "eval_loss": 3.1084439754486084,
+      "eval_perplexity": 22.38618383417341,
+      "eval_runtime": 20.2407,
+      "eval_samples_per_second": 56.372,
+      "eval_steps_per_second": 3.557,
+      "step": 16500
+    },
+    {
+      "epoch": 4.659087719298245,
+      "grad_norm": 0.8326128721237183,
+      "learning_rate": 7.548817767795871e-06,
+      "loss": 2.9335,
+      "step": 16600
+    },
+    {
+      "epoch": 4.687157894736842,
+      "grad_norm": 0.822635293006897,
+      "learning_rate": 6.924948530787947e-06,
+      "loss": 2.9389,
+      "step": 16700
+    },
+    {
+      "epoch": 4.7152280701754385,
+      "grad_norm": 0.8398991227149963,
+      "learning_rate": 6.301079293780024e-06,
+      "loss": 2.9219,
+      "step": 16800
+    },
+    {
+      "epoch": 4.743298245614035,
+      "grad_norm": 0.8308337330818176,
+      "learning_rate": 5.677210056772101e-06,
+      "loss": 2.9374,
+      "step": 16900
+    },
+    {
+      "epoch": 4.771368421052632,
+      "grad_norm": 0.8378835320472717,
+      "learning_rate": 5.053340819764177e-06,
+      "loss": 2.9318,
+      "step": 17000
+    },
+    {
+      "epoch": 4.771368421052632,
+      "eval_accuracy": 0.4270327601022238,
+      "eval_bleu": 0.14678838838741357,
+      "eval_loss": 3.104135036468506,
+      "eval_perplexity": 22.289930657494395,
+      "eval_runtime": 10.9973,
+      "eval_samples_per_second": 103.753,
+      "eval_steps_per_second": 6.547,
+      "step": 17000
+    },
+    {
+      "epoch": 4.799438596491228,
+      "grad_norm": 0.8561663031578064,
+      "learning_rate": 4.429471582756255e-06,
+      "loss": 2.9288,
+      "step": 17100
+    },
+    {
+      "epoch": 4.827508771929825,
+      "grad_norm": 0.8296898007392883,
+      "learning_rate": 3.805602345748331e-06,
+      "loss": 2.9205,
+      "step": 17200
+    },
+    {
+      "epoch": 4.855578947368421,
+      "grad_norm": 0.8513033390045166,
+      "learning_rate": 3.1817331087404082e-06,
+      "loss": 2.9328,
+      "step": 17300
+    },
+    {
+      "epoch": 4.883649122807018,
+      "grad_norm": 0.8511661887168884,
+      "learning_rate": 2.557863871732485e-06,
+      "loss": 2.9392,
+      "step": 17400
+    },
+    {
+      "epoch": 4.911719298245614,
+      "grad_norm": 0.8221142292022705,
+      "learning_rate": 1.933994634724562e-06,
+      "loss": 2.9268,
+      "step": 17500
+    },
+    {
+      "epoch": 4.911719298245614,
+      "eval_accuracy": 0.4273857285929322,
+      "eval_bleu": 0.1460515137764717,
+      "eval_loss": 3.101016044616699,
+      "eval_perplexity": 22.220516852214956,
+      "eval_runtime": 10.9797,
+      "eval_samples_per_second": 103.919,
+      "eval_steps_per_second": 6.558,
+      "step": 17500
+    },
+    {
+      "epoch": 4.93978947368421,
+      "grad_norm": 0.8456584811210632,
+      "learning_rate": 1.3101253977166387e-06,
+      "loss": 2.9326,
+      "step": 17600
+    },
+    {
+      "epoch": 4.967859649122807,
+      "grad_norm": 0.8384252786636353,
+      "learning_rate": 6.862561607087155e-07,
+      "loss": 2.9198,
+      "step": 17700
+    },
+    {
+      "epoch": 4.995929824561403,
+      "grad_norm": 0.8392898440361023,
+      "learning_rate": 6.238692370079231e-08,
+      "loss": 2.9187,
+      "step": 17800
+    },
+    {
+      "epoch": 4.998736842105263,
+      "step": 17810,
+      "total_flos": 1.0584067483285586e+18,
+      "train_loss": 3.584141966119902,
+      "train_runtime": 28526.973,
+      "train_samples_per_second": 19.98,
+      "train_steps_per_second": 0.624
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 17810,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0584067483285586e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9021a79bea00a9b7aa26bd30327d128bc45e546b43d31b5f5131014fecd8912
+oid sha256:a22d41e9776c70f9fbe14b433cb94d3c8f4c35c67d89a20c84239b589a50ea0d
 size 5496