Add checkpoint-3600

.gitattributes

@@ -50,3 +50,4 @@ checkpoints/checkpoint-2800/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/checkpoint-3200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/checkpoint-3400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/checkpoint-3600/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoints/checkpoint-3600/README.md

@@ -0,0 +1,210 @@
+---
+base_model: OpenPathAI/Orbit-3.5-Nano
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:OpenPathAI/Orbit-3.5-Nano
+- lora
+- sft
+- transformers
+- trl
+- unsloth
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

checkpoints/checkpoint-3600/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": {
+    "base_model_class": "Lfm2ForCausalLM",
+    "parent_library": "transformers.models.lfm2.modeling_lfm2",
+    "unsloth_fixed": true
+  },
+  "base_model_name_or_path": "OpenPathAI/Orbit-3.5-Nano",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(?:.*?(?:language|text).*?(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?\\.(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj))|(?:\\bmodel\\.layers\\.[\\d]{1,}(?!.*\\.experts\\.\\d)\\..*?(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?\\.(?:(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)))",
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

checkpoints/checkpoint-3600/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f49f9166b3ae374b0d06afcba06dd5e57a40c543f2e6096fdac1685bee2504
+size 4724960

checkpoints/checkpoint-3600/chat_template.jinja

@@ -0,0 +1,125 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+
+{%- macro format_arg_value(arg_value) -%}
+    {%- if arg_value is string -%}
+        {{- "'" + (arg_value | replace("\\", "\\\\") | replace("'", "\\'") | replace("\n", "\\n") | replace("\r", "\\r")) + "'" -}}
+    {%- elif arg_value is mapping or arg_value is iterable -%}
+        {{- arg_value | tojson -}}
+    {%- else -%}
+        {{- arg_value | string -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping -%}
+        {{- content | tojson -}}
+    {%- elif content is iterable -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item is string -%}
+                {%- set _ns.result = _ns.result + item -%}
+            {%- elif item is mapping and item.get("type") == "image" -%}
+                {%- set _ns.result = _ns.result + "<image>" -%}
+            {%- elif item is mapping and item.get("type") == "text" -%}
+                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + (item | tojson) -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- set tool_calls_ns = namespace(tool_calls=[]) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+        {%- set func_name = func["name"] -%}
+        {%- set func_args = func.get("arguments") -%}
+        {%- set args_ns = namespace(arg_strings=[]) -%}
+        {%- if func_args is mapping -%}
+            {%- for arg_name, arg_value in func_args.items() -%}
+                {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%}
+            {%- endfor -%}
+        {%- elif func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+        {%- endif -%}
+        {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%}
+    {%- endfor -%}
+    {{- "<|tool_call_start|>[" + (tool_calls_ns.tool_calls | join(", ")) + "]<|tool_call_end|>" -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
+    {%- for tool in tools -%}
+        {%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+        {%- endif -%}
+        {%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- set ns.system_prompt = ns.system_prompt + "]" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- for message in messages -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+        {%- set thinking = message.thinking or message.reasoning or message.reasoning_content -%}
+        {%- set thinking = thinking if thinking is string else "" -%}
+        {%- if thinking and keep_thinking -%}
+            {{- "<think>" + thinking + "</think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- set content = "" -%}
+        {%- if message.get("content") -%}
+            {%- set content = parse_content(message.content) -%}
+        {%- endif -%}
+        {%- if not keep_thinking and "</think>" in content -%}
+            {%- set content = content.split("</think>")[-1] | trim -%}
+        {%- endif -%}
+        {%- if content.endswith(_cfm_tag) -%}
+            {%- set _has_cfm = true -%}
+            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+            {%- set content = content[:_trunc_len] -%}
+        {%- endif -%}
+        {{- content -}}
+        {%- if message.tool_calls -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n<think>" -}}
+{%- endif -%}

checkpoints/checkpoint-3600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a3049c4fef2e08681140265cc5bd7f1b5518676616d0f766635dd9efb280af
+size 2455755

checkpoints/checkpoint-3600/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f8a77dff0731a81dd8177ce69f993a01ac6bb4ac0c8c07fecb3d9a8e8effc2a
+size 14645

checkpoints/checkpoint-3600/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ccbd8418d08c90e8516842f5f289aa7e13c2d9c44c18df5a06fdacb9af36a2d
+size 1383

checkpoints/checkpoint-3600/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:449fcf583121070681101e294c260801e1696f262a4a9e8665649cb4e08e8ed0
+size 1465

checkpoints/checkpoint-3600/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:695be7802a0e4b8a81048f0ff5ebb7fc811a0ba5a6be63dbb24deb5a81096f41
+size 17905598

checkpoints/checkpoint-3600/tokenizer_config.json

@@ -0,0 +1,1008 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "legacy": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "added_tokens_decoder": {
+    "124893": {
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124894": {
+      "content": "<|startoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124895": {
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124896": {
+      "content": "<|fim_pre|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124897": {
+      "content": "<|fim_mid|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124898": {
+      "content": "<|fim_suf|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124899": {
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124900": {
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124901": {
+      "content": "<think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "124902": {
+      "content": "</think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "124903": {
+      "content": "<|tool_list_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124904": {
+      "content": "<|tool_list_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124905": {
+      "content": "<|tool_call_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "124906": {
+      "content": "<|tool_call_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "124907": {
+      "content": "<image>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124908": {
+      "content": "<|img_row_1_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124909": {
+      "content": "<|img_row_1_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124910": {
+      "content": "<|img_row_1_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124911": {
+      "content": "<|img_row_1_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124912": {
+      "content": "<|img_row_1_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124913": {
+      "content": "<|img_row_1_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124914": {
+      "content": "<|img_row_1_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124915": {
+      "content": "<|img_row_1_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124916": {
+      "content": "<|img_row_1_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124917": {
+      "content": "<|img_row_1_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124918": {
+      "content": "<|img_row_2_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124919": {
+      "content": "<|img_row_2_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124920": {
+      "content": "<|img_row_2_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124921": {
+      "content": "<|img_row_2_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124922": {
+      "content": "<|img_row_2_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124923": {
+      "content": "<|img_row_2_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124924": {
+      "content": "<|img_row_2_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124925": {
+      "content": "<|img_row_2_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124926": {
+      "content": "<|img_row_2_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124927": {
+      "content": "<|img_row_2_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124928": {
+      "content": "<|img_row_3_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124929": {
+      "content": "<|img_row_3_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124930": {
+      "content": "<|img_row_3_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124931": {
+      "content": "<|img_row_3_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124932": {
+      "content": "<|img_row_3_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124933": {
+      "content": "<|img_row_3_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124934": {
+      "content": "<|img_row_3_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124935": {
+      "content": "<|img_row_3_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124936": {
+      "content": "<|img_row_3_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124937": {
+      "content": "<|img_row_3_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124938": {
+      "content": "<|img_row_4_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124939": {
+      "content": "<|img_row_4_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124940": {
+      "content": "<|img_row_4_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124941": {
+      "content": "<|img_row_4_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124942": {
+      "content": "<|img_row_4_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124943": {
+      "content": "<|img_row_4_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124944": {
+      "content": "<|img_row_4_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124945": {
+      "content": "<|img_row_4_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124946": {
+      "content": "<|img_row_4_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124947": {
+      "content": "<|img_row_4_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124948": {
+      "content": "<|img_row_5_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124949": {
+      "content": "<|img_row_5_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124950": {
+      "content": "<|img_row_5_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124951": {
+      "content": "<|img_row_5_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124952": {
+      "content": "<|img_row_5_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124953": {
+      "content": "<|img_row_5_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124954": {
+      "content": "<|img_row_5_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124955": {
+      "content": "<|img_row_5_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124956": {
+      "content": "<|img_row_5_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124957": {
+      "content": "<|img_row_5_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124958": {
+      "content": "<|img_row_6_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124959": {
+      "content": "<|img_row_6_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124960": {
+      "content": "<|img_row_6_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124961": {
+      "content": "<|img_row_6_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124962": {
+      "content": "<|img_row_6_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124963": {
+      "content": "<|img_row_6_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124964": {
+      "content": "<|img_row_6_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124965": {
+      "content": "<|img_row_6_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124966": {
+      "content": "<|img_row_6_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124967": {
+      "content": "<|img_row_6_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124968": {
+      "content": "<|img_row_7_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124969": {
+      "content": "<|img_row_7_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124970": {
+      "content": "<|img_row_7_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124971": {
+      "content": "<|img_row_7_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124972": {
+      "content": "<|img_row_7_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124973": {
+      "content": "<|img_row_7_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124974": {
+      "content": "<|img_row_7_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124975": {
+      "content": "<|img_row_7_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124976": {
+      "content": "<|img_row_7_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124977": {
+      "content": "<|img_row_7_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124978": {
+      "content": "<|img_row_8_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124979": {
+      "content": "<|img_row_8_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124980": {
+      "content": "<|img_row_8_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124981": {
+      "content": "<|img_row_8_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124982": {
+      "content": "<|img_row_8_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124983": {
+      "content": "<|img_row_8_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124984": {
+      "content": "<|img_row_8_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124985": {
+      "content": "<|img_row_8_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124986": {
+      "content": "<|img_row_8_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124987": {
+      "content": "<|img_row_8_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124988": {
+      "content": "<|img_row_9_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124989": {
+      "content": "<|img_row_9_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124990": {
+      "content": "<|img_row_9_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124991": {
+      "content": "<|img_row_9_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124992": {
+      "content": "<|img_row_9_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124993": {
+      "content": "<|img_row_9_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124994": {
+      "content": "<|img_row_9_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124995": {
+      "content": "<|img_row_9_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124996": {
+      "content": "<|img_row_9_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124997": {
+      "content": "<|img_row_9_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124998": {
+      "content": "<|img_row_10_col_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "124999": {
+      "content": "<|img_row_10_col_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125000": {
+      "content": "<|img_row_10_col_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125001": {
+      "content": "<|img_row_10_col_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125002": {
+      "content": "<|img_row_10_col_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125003": {
+      "content": "<|img_row_10_col_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125004": {
+      "content": "<|img_row_10_col_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125005": {
+      "content": "<|img_row_10_col_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125006": {
+      "content": "<|img_row_10_col_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125007": {
+      "content": "<|img_row_10_col_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125008": {
+      "content": "<|img_thumbnail|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125009": {
+      "content": "<|image_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125010": {
+      "content": "<|image_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125011": {
+      "content": "<|image_split|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125012": {
+      "content": "<|audio_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125013": {
+      "content": "<|text_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125014": {
+      "content": "<|text_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125015": {
+      "content": "<|mixed_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "125016": {
+      "content": "<|mixed_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  }
+}

checkpoints/checkpoint-3600/trainer_state.json

@@ -0,0 +1,2554 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5139369713408758,
+  "eval_steps": 500,
+  "global_step": 3600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0014276026981690996,
+      "grad_norm": 1.2670224905014038,
+      "learning_rate": 0.00016,
+      "loss": 2.898906135559082,
+      "step": 10
+    },
+    {
+      "epoch": 0.0028552053963381992,
+      "grad_norm": 1.2538419961929321,
+      "learning_rate": 0.00019992382766008095,
+      "loss": 2.378516960144043,
+      "step": 20
+    },
+    {
+      "epoch": 0.004282808094507299,
+      "grad_norm": 1.409246563911438,
+      "learning_rate": 0.0001998286122351821,
+      "loss": 2.4753650665283202,
+      "step": 30
+    },
+    {
+      "epoch": 0.0057104107926763984,
+      "grad_norm": 1.094603180885315,
+      "learning_rate": 0.00019973339681028326,
+      "loss": 2.2721933364868163,
+      "step": 40
+    },
+    {
+      "epoch": 0.007138013490845497,
+      "grad_norm": 1.1403776407241821,
+      "learning_rate": 0.00019963818138538444,
+      "loss": 2.263425827026367,
+      "step": 50
+    },
+    {
+      "epoch": 0.008565616189014597,
+      "grad_norm": 1.1656428575515747,
+      "learning_rate": 0.0001995429659604856,
+      "loss": 2.363163185119629,
+      "step": 60
+    },
+    {
+      "epoch": 0.009993218887183696,
+      "grad_norm": 1.1926625967025757,
+      "learning_rate": 0.00019944775053558678,
+      "loss": 2.1333871841430665,
+      "step": 70
+    },
+    {
+      "epoch": 0.011420821585352797,
+      "grad_norm": 0.9967195391654968,
+      "learning_rate": 0.00019935253511068794,
+      "loss": 2.279196929931641,
+      "step": 80
+    },
+    {
+      "epoch": 0.012848424283521896,
+      "grad_norm": 1.1345535516738892,
+      "learning_rate": 0.00019925731968578912,
+      "loss": 2.181261444091797,
+      "step": 90
+    },
+    {
+      "epoch": 0.014276026981690995,
+      "grad_norm": 1.2285407781600952,
+      "learning_rate": 0.00019916210426089027,
+      "loss": 2.2282310485839845,
+      "step": 100
+    },
+    {
+      "epoch": 0.015703629679860096,
+      "grad_norm": 2.191948890686035,
+      "learning_rate": 0.00019906688883599146,
+      "loss": 2.2829305648803713,
+      "step": 110
+    },
+    {
+      "epoch": 0.017131232378029194,
+      "grad_norm": 1.1868839263916016,
+      "learning_rate": 0.00019897167341109258,
+      "loss": 2.1634748458862303,
+      "step": 120
+    },
+    {
+      "epoch": 0.018558835076198293,
+      "grad_norm": 1.3730543851852417,
+      "learning_rate": 0.00019887645798619377,
+      "loss": 2.235073471069336,
+      "step": 130
+    },
+    {
+      "epoch": 0.019986437774367392,
+      "grad_norm": 1.3060146570205688,
+      "learning_rate": 0.00019878124256129492,
+      "loss": 2.254693031311035,
+      "step": 140
+    },
+    {
+      "epoch": 0.021414040472536495,
+      "grad_norm": 1.13761305809021,
+      "learning_rate": 0.0001986860271363961,
+      "loss": 2.255308723449707,
+      "step": 150
+    },
+    {
+      "epoch": 0.022841643170705594,
+      "grad_norm": 1.2420964241027832,
+      "learning_rate": 0.00019859081171149726,
+      "loss": 2.199342155456543,
+      "step": 160
+    },
+    {
+      "epoch": 0.024269245868874693,
+      "grad_norm": 1.2628268003463745,
+      "learning_rate": 0.00019849559628659844,
+      "loss": 2.2568809509277346,
+      "step": 170
+    },
+    {
+      "epoch": 0.02569684856704379,
+      "grad_norm": 1.0423402786254883,
+      "learning_rate": 0.0001984003808616996,
+      "loss": 2.201403045654297,
+      "step": 180
+    },
+    {
+      "epoch": 0.02712445126521289,
+      "grad_norm": 1.3187193870544434,
+      "learning_rate": 0.00019830516543680078,
+      "loss": 2.31024055480957,
+      "step": 190
+    },
+    {
+      "epoch": 0.02855205396338199,
+      "grad_norm": 1.287013292312622,
+      "learning_rate": 0.00019820995001190194,
+      "loss": 2.2775556564331056,
+      "step": 200
+    },
+    {
+      "epoch": 0.029979656661551092,
+      "grad_norm": 1.445034384727478,
+      "learning_rate": 0.00019811473458700312,
+      "loss": 2.214271354675293,
+      "step": 210
+    },
+    {
+      "epoch": 0.03140725935972019,
+      "grad_norm": 1.4336904287338257,
+      "learning_rate": 0.00019801951916210427,
+      "loss": 2.1211181640625,
+      "step": 220
+    },
+    {
+      "epoch": 0.03283486205788929,
+      "grad_norm": 1.303303837776184,
+      "learning_rate": 0.00019792430373720543,
+      "loss": 2.1037008285522463,
+      "step": 230
+    },
+    {
+      "epoch": 0.03426246475605839,
+      "grad_norm": 1.2367061376571655,
+      "learning_rate": 0.0001978290883123066,
+      "loss": 2.1291038513183596,
+      "step": 240
+    },
+    {
+      "epoch": 0.03569006745422749,
+      "grad_norm": 1.314647912979126,
+      "learning_rate": 0.00019773387288740777,
+      "loss": 2.1351301193237306,
+      "step": 250
+    },
+    {
+      "epoch": 0.03711767015239659,
+      "grad_norm": 1.3084362745285034,
+      "learning_rate": 0.00019763865746250895,
+      "loss": 2.0980342864990233,
+      "step": 260
+    },
+    {
+      "epoch": 0.038545272850565686,
+      "grad_norm": 1.712534785270691,
+      "learning_rate": 0.0001975434420376101,
+      "loss": 2.25567569732666,
+      "step": 270
+    },
+    {
+      "epoch": 0.039972875548734785,
+      "grad_norm": 1.5107096433639526,
+      "learning_rate": 0.0001974482266127113,
+      "loss": 2.198653793334961,
+      "step": 280
+    },
+    {
+      "epoch": 0.041400478246903884,
+      "grad_norm": 1.1626437902450562,
+      "learning_rate": 0.00019735301118781244,
+      "loss": 2.136816215515137,
+      "step": 290
+    },
+    {
+      "epoch": 0.04282808094507299,
+      "grad_norm": 1.551853060722351,
+      "learning_rate": 0.00019725779576291362,
+      "loss": 2.2106935501098635,
+      "step": 300
+    },
+    {
+      "epoch": 0.04425568364324209,
+      "grad_norm": 1.5131734609603882,
+      "learning_rate": 0.00019716258033801475,
+      "loss": 2.0967830657958983,
+      "step": 310
+    },
+    {
+      "epoch": 0.04568328634141119,
+      "grad_norm": 1.310641884803772,
+      "learning_rate": 0.00019706736491311593,
+      "loss": 2.2105255126953125,
+      "step": 320
+    },
+    {
+      "epoch": 0.047110889039580287,
+      "grad_norm": 1.3621881008148193,
+      "learning_rate": 0.0001969721494882171,
+      "loss": 2.1775463104248045,
+      "step": 330
+    },
+    {
+      "epoch": 0.048538491737749385,
+      "grad_norm": 1.4618415832519531,
+      "learning_rate": 0.00019687693406331827,
+      "loss": 2.148931694030762,
+      "step": 340
+    },
+    {
+      "epoch": 0.049966094435918484,
+      "grad_norm": 1.3173977136611938,
+      "learning_rate": 0.00019678171863841943,
+      "loss": 2.1974794387817385,
+      "step": 350
+    },
+    {
+      "epoch": 0.05139369713408758,
+      "grad_norm": 1.1830055713653564,
+      "learning_rate": 0.0001966865032135206,
+      "loss": 2.1695978164672853,
+      "step": 360
+    },
+    {
+      "epoch": 0.05282129983225668,
+      "grad_norm": 1.1831724643707275,
+      "learning_rate": 0.00019659128778862177,
+      "loss": 2.0823343276977537,
+      "step": 370
+    },
+    {
+      "epoch": 0.05424890253042578,
+      "grad_norm": 1.1693743467330933,
+      "learning_rate": 0.00019649607236372295,
+      "loss": 2.1867488861083983,
+      "step": 380
+    },
+    {
+      "epoch": 0.05567650522859488,
+      "grad_norm": 1.25830078125,
+      "learning_rate": 0.0001964008569388241,
+      "loss": 2.2133615493774412,
+      "step": 390
+    },
+    {
+      "epoch": 0.05710410792676398,
+      "grad_norm": 1.3678141832351685,
+      "learning_rate": 0.00019630564151392529,
+      "loss": 2.308313751220703,
+      "step": 400
+    },
+    {
+      "epoch": 0.05853171062493308,
+      "grad_norm": 1.1440492868423462,
+      "learning_rate": 0.00019621042608902641,
+      "loss": 2.1140726089477537,
+      "step": 410
+    },
+    {
+      "epoch": 0.059959313323102184,
+      "grad_norm": 1.1869196891784668,
+      "learning_rate": 0.0001961152106641276,
+      "loss": 2.0346582412719725,
+      "step": 420
+    },
+    {
+      "epoch": 0.06138691602127128,
+      "grad_norm": 1.4892522096633911,
+      "learning_rate": 0.00019601999523922875,
+      "loss": 2.2025136947631836,
+      "step": 430
+    },
+    {
+      "epoch": 0.06281451871944038,
+      "grad_norm": 1.8906227350234985,
+      "learning_rate": 0.00019592477981432993,
+      "loss": 2.297587776184082,
+      "step": 440
+    },
+    {
+      "epoch": 0.06424212141760947,
+      "grad_norm": 1.2389851808547974,
+      "learning_rate": 0.0001958295643894311,
+      "loss": 2.130488395690918,
+      "step": 450
+    },
+    {
+      "epoch": 0.06566972411577858,
+      "grad_norm": 1.425410509109497,
+      "learning_rate": 0.00019573434896453227,
+      "loss": 2.144012451171875,
+      "step": 460
+    },
+    {
+      "epoch": 0.06709732681394767,
+      "grad_norm": 1.3087464570999146,
+      "learning_rate": 0.00019563913353963343,
+      "loss": 2.2011600494384767,
+      "step": 470
+    },
+    {
+      "epoch": 0.06852492951211678,
+      "grad_norm": 1.3441588878631592,
+      "learning_rate": 0.0001955439181147346,
+      "loss": 2.291761016845703,
+      "step": 480
+    },
+    {
+      "epoch": 0.06995253221028588,
+      "grad_norm": 1.2851412296295166,
+      "learning_rate": 0.00019544870268983577,
+      "loss": 2.24096622467041,
+      "step": 490
+    },
+    {
+      "epoch": 0.07138013490845498,
+      "grad_norm": 1.2628257274627686,
+      "learning_rate": 0.00019535348726493692,
+      "loss": 2.080752944946289,
+      "step": 500
+    },
+    {
+      "epoch": 0.07280773760662408,
+      "grad_norm": 1.2638740539550781,
+      "learning_rate": 0.00019525827184003808,
+      "loss": 2.4698177337646485,
+      "step": 510
+    },
+    {
+      "epoch": 0.07423534030479317,
+      "grad_norm": 1.2560440301895142,
+      "learning_rate": 0.00019516305641513926,
+      "loss": 2.132457733154297,
+      "step": 520
+    },
+    {
+      "epoch": 0.07566294300296228,
+      "grad_norm": 1.7629936933517456,
+      "learning_rate": 0.0001950678409902404,
+      "loss": 2.166901969909668,
+      "step": 530
+    },
+    {
+      "epoch": 0.07709054570113137,
+      "grad_norm": 1.8175091743469238,
+      "learning_rate": 0.0001949726255653416,
+      "loss": 2.1663414001464845,
+      "step": 540
+    },
+    {
+      "epoch": 0.07851814839930048,
+      "grad_norm": 1.1744414567947388,
+      "learning_rate": 0.00019487741014044275,
+      "loss": 2.388395309448242,
+      "step": 550
+    },
+    {
+      "epoch": 0.07994575109746957,
+      "grad_norm": 0.9943644404411316,
+      "learning_rate": 0.00019478219471554393,
+      "loss": 2.1379253387451174,
+      "step": 560
+    },
+    {
+      "epoch": 0.08137335379563868,
+      "grad_norm": 1.2485833168029785,
+      "learning_rate": 0.0001946869792906451,
+      "loss": 2.1750890731811525,
+      "step": 570
+    },
+    {
+      "epoch": 0.08280095649380777,
+      "grad_norm": 1.0852699279785156,
+      "learning_rate": 0.00019459176386574627,
+      "loss": 2.2631216049194336,
+      "step": 580
+    },
+    {
+      "epoch": 0.08422855919197687,
+      "grad_norm": 1.5462923049926758,
+      "learning_rate": 0.00019449654844084743,
+      "loss": 2.2366138458251954,
+      "step": 590
+    },
+    {
+      "epoch": 0.08565616189014598,
+      "grad_norm": 1.2923717498779297,
+      "learning_rate": 0.00019440133301594858,
+      "loss": 2.1567543029785154,
+      "step": 600
+    },
+    {
+      "epoch": 0.08708376458831507,
+      "grad_norm": 1.3011770248413086,
+      "learning_rate": 0.00019430611759104976,
+      "loss": 2.1422496795654298,
+      "step": 610
+    },
+    {
+      "epoch": 0.08851136728648418,
+      "grad_norm": 1.4617754220962524,
+      "learning_rate": 0.00019421090216615092,
+      "loss": 2.1605356216430662,
+      "step": 620
+    },
+    {
+      "epoch": 0.08993896998465327,
+      "grad_norm": 1.5295599699020386,
+      "learning_rate": 0.0001941156867412521,
+      "loss": 2.0025182723999024,
+      "step": 630
+    },
+    {
+      "epoch": 0.09136657268282238,
+      "grad_norm": 1.3012663125991821,
+      "learning_rate": 0.00019402047131635326,
+      "loss": 2.1372703552246093,
+      "step": 640
+    },
+    {
+      "epoch": 0.09279417538099147,
+      "grad_norm": 1.4276515245437622,
+      "learning_rate": 0.00019392525589145444,
+      "loss": 2.315623474121094,
+      "step": 650
+    },
+    {
+      "epoch": 0.09422177807916057,
+      "grad_norm": 1.5132569074630737,
+      "learning_rate": 0.0001938300404665556,
+      "loss": 2.258671760559082,
+      "step": 660
+    },
+    {
+      "epoch": 0.09564938077732967,
+      "grad_norm": 1.1320735216140747,
+      "learning_rate": 0.00019373482504165678,
+      "loss": 2.21956844329834,
+      "step": 670
+    },
+    {
+      "epoch": 0.09707698347549877,
+      "grad_norm": 1.1464308500289917,
+      "learning_rate": 0.00019363960961675793,
+      "loss": 2.173979949951172,
+      "step": 680
+    },
+    {
+      "epoch": 0.09850458617366786,
+      "grad_norm": 1.2780959606170654,
+      "learning_rate": 0.0001935443941918591,
+      "loss": 2.159123992919922,
+      "step": 690
+    },
+    {
+      "epoch": 0.09993218887183697,
+      "grad_norm": 1.3222688436508179,
+      "learning_rate": 0.00019344917876696024,
+      "loss": 2.1596654891967773,
+      "step": 700
+    },
+    {
+      "epoch": 0.10135979157000606,
+      "grad_norm": 1.2251396179199219,
+      "learning_rate": 0.00019335396334206143,
+      "loss": 2.0882038116455077,
+      "step": 710
+    },
+    {
+      "epoch": 0.10278739426817517,
+      "grad_norm": 1.2655571699142456,
+      "learning_rate": 0.00019325874791716258,
+      "loss": 2.1701095581054686,
+      "step": 720
+    },
+    {
+      "epoch": 0.10421499696634427,
+      "grad_norm": 1.3139557838439941,
+      "learning_rate": 0.00019316353249226376,
+      "loss": 2.1374244689941406,
+      "step": 730
+    },
+    {
+      "epoch": 0.10564259966451336,
+      "grad_norm": 0.9476764798164368,
+      "learning_rate": 0.00019306831706736492,
+      "loss": 2.131810760498047,
+      "step": 740
+    },
+    {
+      "epoch": 0.10707020236268247,
+      "grad_norm": 1.3916528224945068,
+      "learning_rate": 0.0001929731016424661,
+      "loss": 2.2276470184326174,
+      "step": 750
+    },
+    {
+      "epoch": 0.10849780506085156,
+      "grad_norm": 1.2913047075271606,
+      "learning_rate": 0.00019287788621756726,
+      "loss": 2.0985342025756837,
+      "step": 760
+    },
+    {
+      "epoch": 0.10992540775902067,
+      "grad_norm": 1.4780009984970093,
+      "learning_rate": 0.00019278267079266844,
+      "loss": 2.3402137756347656,
+      "step": 770
+    },
+    {
+      "epoch": 0.11135301045718976,
+      "grad_norm": 1.38652765750885,
+      "learning_rate": 0.0001926874553677696,
+      "loss": 2.1956760406494142,
+      "step": 780
+    },
+    {
+      "epoch": 0.11278061315535887,
+      "grad_norm": 1.2437113523483276,
+      "learning_rate": 0.00019259223994287075,
+      "loss": 2.344030570983887,
+      "step": 790
+    },
+    {
+      "epoch": 0.11420821585352796,
+      "grad_norm": 1.1223382949829102,
+      "learning_rate": 0.0001924970245179719,
+      "loss": 2.245418167114258,
+      "step": 800
+    },
+    {
+      "epoch": 0.11563581855169706,
+      "grad_norm": 1.5828821659088135,
+      "learning_rate": 0.0001924018090930731,
+      "loss": 2.2958986282348635,
+      "step": 810
+    },
+    {
+      "epoch": 0.11706342124986616,
+      "grad_norm": 1.1058349609375,
+      "learning_rate": 0.00019230659366817424,
+      "loss": 2.0216651916503907,
+      "step": 820
+    },
+    {
+      "epoch": 0.11849102394803526,
+      "grad_norm": 1.1991469860076904,
+      "learning_rate": 0.00019221137824327543,
+      "loss": 2.1821861267089844,
+      "step": 830
+    },
+    {
+      "epoch": 0.11991862664620437,
+      "grad_norm": 1.0211104154586792,
+      "learning_rate": 0.00019211616281837658,
+      "loss": 2.1515743255615236,
+      "step": 840
+    },
+    {
+      "epoch": 0.12134622934437346,
+      "grad_norm": 1.163668155670166,
+      "learning_rate": 0.00019202094739347776,
+      "loss": 2.0592647552490235,
+      "step": 850
+    },
+    {
+      "epoch": 0.12277383204254257,
+      "grad_norm": 1.0804855823516846,
+      "learning_rate": 0.00019192573196857892,
+      "loss": 2.0916107177734373,
+      "step": 860
+    },
+    {
+      "epoch": 0.12420143474071166,
+      "grad_norm": 1.2400697469711304,
+      "learning_rate": 0.0001918305165436801,
+      "loss": 2.0994134902954102,
+      "step": 870
+    },
+    {
+      "epoch": 0.12562903743888076,
+      "grad_norm": 1.3643152713775635,
+      "learning_rate": 0.00019173530111878123,
+      "loss": 2.2519294738769533,
+      "step": 880
+    },
+    {
+      "epoch": 0.12705664013704987,
+      "grad_norm": 1.3758548498153687,
+      "learning_rate": 0.0001916400856938824,
+      "loss": 2.22522029876709,
+      "step": 890
+    },
+    {
+      "epoch": 0.12848424283521895,
+      "grad_norm": 1.1675829887390137,
+      "learning_rate": 0.00019154487026898357,
+      "loss": 2.1150989532470703,
+      "step": 900
+    },
+    {
+      "epoch": 0.12991184553338805,
+      "grad_norm": 1.2862130403518677,
+      "learning_rate": 0.00019144965484408475,
+      "loss": 2.289804458618164,
+      "step": 910
+    },
+    {
+      "epoch": 0.13133944823155716,
+      "grad_norm": 1.271008849143982,
+      "learning_rate": 0.0001913639609616758,
+      "loss": 2.1846010208129885,
+      "step": 920
+    },
+    {
+      "epoch": 0.13276705092972627,
+      "grad_norm": 1.1630209684371948,
+      "learning_rate": 0.00019126874553677695,
+      "loss": 2.2126953125,
+      "step": 930
+    },
+    {
+      "epoch": 0.13419465362789534,
+      "grad_norm": 1.1368460655212402,
+      "learning_rate": 0.00019117353011187813,
+      "loss": 2.178596305847168,
+      "step": 940
+    },
+    {
+      "epoch": 0.13562225632606445,
+      "grad_norm": 1.0295072793960571,
+      "learning_rate": 0.0001910783146869793,
+      "loss": 2.2031627655029298,
+      "step": 950
+    },
+    {
+      "epoch": 0.13704985902423356,
+      "grad_norm": 1.379927396774292,
+      "learning_rate": 0.00019098309926208047,
+      "loss": 2.120400619506836,
+      "step": 960
+    },
+    {
+      "epoch": 0.13847746172240266,
+      "grad_norm": 1.1450871229171753,
+      "learning_rate": 0.00019088788383718163,
+      "loss": 2.151012992858887,
+      "step": 970
+    },
+    {
+      "epoch": 0.13990506442057177,
+      "grad_norm": 1.0418351888656616,
+      "learning_rate": 0.0001907926684122828,
+      "loss": 2.1239688873291014,
+      "step": 980
+    },
+    {
+      "epoch": 0.14133266711874085,
+      "grad_norm": 1.0744861364364624,
+      "learning_rate": 0.00019069745298738397,
+      "loss": 2.090304374694824,
+      "step": 990
+    },
+    {
+      "epoch": 0.14276026981690995,
+      "grad_norm": 1.1106981039047241,
+      "learning_rate": 0.00019060223756248515,
+      "loss": 2.225258636474609,
+      "step": 1000
+    },
+    {
+      "epoch": 0.14418787251507906,
+      "grad_norm": 1.4493266344070435,
+      "learning_rate": 0.00019050702213758628,
+      "loss": 2.0656681060791016,
+      "step": 1010
+    },
+    {
+      "epoch": 0.14561547521324816,
+      "grad_norm": 1.1414896249771118,
+      "learning_rate": 0.00019041180671268746,
+      "loss": 2.103784942626953,
+      "step": 1020
+    },
+    {
+      "epoch": 0.14704307791141724,
+      "grad_norm": 1.3252989053726196,
+      "learning_rate": 0.00019031659128778861,
+      "loss": 2.2042461395263673,
+      "step": 1030
+    },
+    {
+      "epoch": 0.14847068060958635,
+      "grad_norm": 1.265684962272644,
+      "learning_rate": 0.0001902213758628898,
+      "loss": 1.9412918090820312,
+      "step": 1040
+    },
+    {
+      "epoch": 0.14989828330775545,
+      "grad_norm": 1.3606458902359009,
+      "learning_rate": 0.00019012616043799095,
+      "loss": 2.187259292602539,
+      "step": 1050
+    },
+    {
+      "epoch": 0.15132588600592456,
+      "grad_norm": 1.0192638635635376,
+      "learning_rate": 0.00019003094501309213,
+      "loss": 2.182966423034668,
+      "step": 1060
+    },
+    {
+      "epoch": 0.15275348870409364,
+      "grad_norm": 1.5140159130096436,
+      "learning_rate": 0.0001899357295881933,
+      "loss": 2.3263607025146484,
+      "step": 1070
+    },
+    {
+      "epoch": 0.15418109140226274,
+      "grad_norm": 1.1377187967300415,
+      "learning_rate": 0.00018984051416329447,
+      "loss": 2.159488868713379,
+      "step": 1080
+    },
+    {
+      "epoch": 0.15560869410043185,
+      "grad_norm": 1.1288865804672241,
+      "learning_rate": 0.00018974529873839563,
+      "loss": 2.306215286254883,
+      "step": 1090
+    },
+    {
+      "epoch": 0.15703629679860096,
+      "grad_norm": 1.280045747756958,
+      "learning_rate": 0.00018965008331349678,
+      "loss": 2.225907897949219,
+      "step": 1100
+    },
+    {
+      "epoch": 0.15846389949677006,
+      "grad_norm": 1.145614743232727,
+      "learning_rate": 0.00018955486788859797,
+      "loss": 2.1122207641601562,
+      "step": 1110
+    },
+    {
+      "epoch": 0.15989150219493914,
+      "grad_norm": 1.1612759828567505,
+      "learning_rate": 0.00018945965246369912,
+      "loss": 2.2271358489990236,
+      "step": 1120
+    },
+    {
+      "epoch": 0.16131910489310824,
+      "grad_norm": 1.0958333015441895,
+      "learning_rate": 0.0001893644370388003,
+      "loss": 2.088473320007324,
+      "step": 1130
+    },
+    {
+      "epoch": 0.16274670759127735,
+      "grad_norm": 1.1749203205108643,
+      "learning_rate": 0.00018926922161390146,
+      "loss": 2.091511917114258,
+      "step": 1140
+    },
+    {
+      "epoch": 0.16417431028944646,
+      "grad_norm": 1.1318912506103516,
+      "learning_rate": 0.00018917400618900264,
+      "loss": 2.2805349349975588,
+      "step": 1150
+    },
+    {
+      "epoch": 0.16560191298761553,
+      "grad_norm": 1.220713496208191,
+      "learning_rate": 0.0001890787907641038,
+      "loss": 2.129801940917969,
+      "step": 1160
+    },
+    {
+      "epoch": 0.16702951568578464,
+      "grad_norm": 1.0876924991607666,
+      "learning_rate": 0.00018898357533920498,
+      "loss": 2.0997352600097656,
+      "step": 1170
+    },
+    {
+      "epoch": 0.16845711838395375,
+      "grad_norm": 1.175884485244751,
+      "learning_rate": 0.00018888835991430613,
+      "loss": 2.0686941146850586,
+      "step": 1180
+    },
+    {
+      "epoch": 0.16988472108212285,
+      "grad_norm": 1.3588541746139526,
+      "learning_rate": 0.00018879314448940732,
+      "loss": 2.2050849914550783,
+      "step": 1190
+    },
+    {
+      "epoch": 0.17131232378029196,
+      "grad_norm": 1.26045560836792,
+      "learning_rate": 0.00018869792906450844,
+      "loss": 2.1561567306518556,
+      "step": 1200
+    },
+    {
+      "epoch": 0.17273992647846104,
+      "grad_norm": 1.0756542682647705,
+      "learning_rate": 0.00018860271363960963,
+      "loss": 2.163359832763672,
+      "step": 1210
+    },
+    {
+      "epoch": 0.17416752917663014,
+      "grad_norm": 1.308608055114746,
+      "learning_rate": 0.00018850749821471078,
+      "loss": 2.209610366821289,
+      "step": 1220
+    },
+    {
+      "epoch": 0.17559513187479925,
+      "grad_norm": 1.369969129562378,
+      "learning_rate": 0.00018841228278981196,
+      "loss": 2.3173370361328125,
+      "step": 1230
+    },
+    {
+      "epoch": 0.17702273457296835,
+      "grad_norm": 1.2242190837860107,
+      "learning_rate": 0.00018831706736491312,
+      "loss": 2.149942398071289,
+      "step": 1240
+    },
+    {
+      "epoch": 0.17845033727113743,
+      "grad_norm": 1.226375937461853,
+      "learning_rate": 0.0001882218519400143,
+      "loss": 2.152383232116699,
+      "step": 1250
+    },
+    {
+      "epoch": 0.17987793996930654,
+      "grad_norm": 2.0897111892700195,
+      "learning_rate": 0.00018812663651511546,
+      "loss": 2.070216751098633,
+      "step": 1260
+    },
+    {
+      "epoch": 0.18130554266747564,
+      "grad_norm": 1.239769697189331,
+      "learning_rate": 0.00018803142109021664,
+      "loss": 2.1863370895385743,
+      "step": 1270
+    },
+    {
+      "epoch": 0.18273314536564475,
+      "grad_norm": 1.2516264915466309,
+      "learning_rate": 0.0001879362056653178,
+      "loss": 2.154579734802246,
+      "step": 1280
+    },
+    {
+      "epoch": 0.18416074806381383,
+      "grad_norm": 1.300251841545105,
+      "learning_rate": 0.00018784099024041895,
+      "loss": 2.084653854370117,
+      "step": 1290
+    },
+    {
+      "epoch": 0.18558835076198293,
+      "grad_norm": 1.5401705503463745,
+      "learning_rate": 0.0001877457748155201,
+      "loss": 2.1652135848999023,
+      "step": 1300
+    },
+    {
+      "epoch": 0.18701595346015204,
+      "grad_norm": 1.388256549835205,
+      "learning_rate": 0.0001876505593906213,
+      "loss": 2.215251350402832,
+      "step": 1310
+    },
+    {
+      "epoch": 0.18844355615832115,
+      "grad_norm": 1.0548882484436035,
+      "learning_rate": 0.00018755534396572244,
+      "loss": 2.057052230834961,
+      "step": 1320
+    },
+    {
+      "epoch": 0.18987115885649025,
+      "grad_norm": 1.318389654159546,
+      "learning_rate": 0.00018746012854082363,
+      "loss": 2.1746826171875,
+      "step": 1330
+    },
+    {
+      "epoch": 0.19129876155465933,
+      "grad_norm": 1.1861886978149414,
+      "learning_rate": 0.00018736491311592478,
+      "loss": 2.107588005065918,
+      "step": 1340
+    },
+    {
+      "epoch": 0.19272636425282844,
+      "grad_norm": 1.0857220888137817,
+      "learning_rate": 0.00018726969769102596,
+      "loss": 2.1044857025146486,
+      "step": 1350
+    },
+    {
+      "epoch": 0.19415396695099754,
+      "grad_norm": 1.144823670387268,
+      "learning_rate": 0.00018717448226612712,
+      "loss": 2.132158660888672,
+      "step": 1360
+    },
+    {
+      "epoch": 0.19558156964916665,
+      "grad_norm": 1.1121206283569336,
+      "learning_rate": 0.0001870792668412283,
+      "loss": 2.1906471252441406,
+      "step": 1370
+    },
+    {
+      "epoch": 0.19700917234733573,
+      "grad_norm": 1.2356815338134766,
+      "learning_rate": 0.00018698405141632946,
+      "loss": 2.2056690216064454,
+      "step": 1380
+    },
+    {
+      "epoch": 0.19843677504550483,
+      "grad_norm": 1.3212710618972778,
+      "learning_rate": 0.0001868888359914306,
+      "loss": 2.168708610534668,
+      "step": 1390
+    },
+    {
+      "epoch": 0.19986437774367394,
+      "grad_norm": 1.1257580518722534,
+      "learning_rate": 0.00018679362056653177,
+      "loss": 2.18591423034668,
+      "step": 1400
+    },
+    {
+      "epoch": 0.20129198044184304,
+      "grad_norm": 1.5445501804351807,
+      "learning_rate": 0.00018669840514163295,
+      "loss": 2.206186294555664,
+      "step": 1410
+    },
+    {
+      "epoch": 0.20271958314001212,
+      "grad_norm": 1.5519709587097168,
+      "learning_rate": 0.0001866031897167341,
+      "loss": 2.1855854034423827,
+      "step": 1420
+    },
+    {
+      "epoch": 0.20414718583818123,
+      "grad_norm": 1.0834031105041504,
+      "learning_rate": 0.0001865079742918353,
+      "loss": 2.1382625579833983,
+      "step": 1430
+    },
+    {
+      "epoch": 0.20557478853635033,
+      "grad_norm": 1.0713154077529907,
+      "learning_rate": 0.00018641275886693644,
+      "loss": 2.1709274291992187,
+      "step": 1440
+    },
+    {
+      "epoch": 0.20700239123451944,
+      "grad_norm": 1.3530168533325195,
+      "learning_rate": 0.00018631754344203763,
+      "loss": 2.062696075439453,
+      "step": 1450
+    },
+    {
+      "epoch": 0.20842999393268855,
+      "grad_norm": 1.0635544061660767,
+      "learning_rate": 0.00018622232801713878,
+      "loss": 2.209213447570801,
+      "step": 1460
+    },
+    {
+      "epoch": 0.20985759663085762,
+      "grad_norm": 1.2547699213027954,
+      "learning_rate": 0.00018612711259223996,
+      "loss": 2.111533546447754,
+      "step": 1470
+    },
+    {
+      "epoch": 0.21128519932902673,
+      "grad_norm": 1.1931551694869995,
+      "learning_rate": 0.00018603189716734112,
+      "loss": 2.3098974227905273,
+      "step": 1480
+    },
+    {
+      "epoch": 0.21271280202719584,
+      "grad_norm": 1.0127021074295044,
+      "learning_rate": 0.00018593668174244227,
+      "loss": 2.1491844177246096,
+      "step": 1490
+    },
+    {
+      "epoch": 0.21414040472536494,
+      "grad_norm": 1.3180660009384155,
+      "learning_rate": 0.00018584146631754346,
+      "loss": 2.188648223876953,
+      "step": 1500
+    },
+    {
+      "epoch": 0.21556800742353402,
+      "grad_norm": 1.1885178089141846,
+      "learning_rate": 0.0001857462508926446,
+      "loss": 2.0914661407470705,
+      "step": 1510
+    },
+    {
+      "epoch": 0.21699561012170313,
+      "grad_norm": 1.1395208835601807,
+      "learning_rate": 0.0001856510354677458,
+      "loss": 2.2511472702026367,
+      "step": 1520
+    },
+    {
+      "epoch": 0.21842321281987223,
+      "grad_norm": 1.4837855100631714,
+      "learning_rate": 0.00018555582004284695,
+      "loss": 2.2735774993896483,
+      "step": 1530
+    },
+    {
+      "epoch": 0.21985081551804134,
+      "grad_norm": 1.203804612159729,
+      "learning_rate": 0.00018546060461794813,
+      "loss": 2.2867109298706056,
+      "step": 1540
+    },
+    {
+      "epoch": 0.22127841821621042,
+      "grad_norm": 0.997947096824646,
+      "learning_rate": 0.0001853653891930493,
+      "loss": 2.184854507446289,
+      "step": 1550
+    },
+    {
+      "epoch": 0.22270602091437952,
+      "grad_norm": 1.484794020652771,
+      "learning_rate": 0.00018527017376815047,
+      "loss": 2.232890510559082,
+      "step": 1560
+    },
+    {
+      "epoch": 0.22413362361254863,
+      "grad_norm": 1.3194350004196167,
+      "learning_rate": 0.00018517495834325162,
+      "loss": 2.0665668487548827,
+      "step": 1570
+    },
+    {
+      "epoch": 0.22556122631071773,
+      "grad_norm": 1.4378708600997925,
+      "learning_rate": 0.00018507974291835278,
+      "loss": 2.025044059753418,
+      "step": 1580
+    },
+    {
+      "epoch": 0.22698882900888684,
+      "grad_norm": 1.161460041999817,
+      "learning_rate": 0.00018498452749345394,
+      "loss": 2.2053255081176757,
+      "step": 1590
+    },
+    {
+      "epoch": 0.22841643170705592,
+      "grad_norm": 1.0879746675491333,
+      "learning_rate": 0.00018488931206855512,
+      "loss": 2.1081918716430663,
+      "step": 1600
+    },
+    {
+      "epoch": 0.22984403440522502,
+      "grad_norm": 1.530381679534912,
+      "learning_rate": 0.00018479409664365627,
+      "loss": 2.135531997680664,
+      "step": 1610
+    },
+    {
+      "epoch": 0.23127163710339413,
+      "grad_norm": 1.2382729053497314,
+      "learning_rate": 0.00018469888121875746,
+      "loss": 2.0582502365112303,
+      "step": 1620
+    },
+    {
+      "epoch": 0.23269923980156323,
+      "grad_norm": 1.2160850763320923,
+      "learning_rate": 0.0001846036657938586,
+      "loss": 2.191323471069336,
+      "step": 1630
+    },
+    {
+      "epoch": 0.2341268424997323,
+      "grad_norm": 1.3606561422348022,
+      "learning_rate": 0.0001845084503689598,
+      "loss": 2.139387512207031,
+      "step": 1640
+    },
+    {
+      "epoch": 0.23555444519790142,
+      "grad_norm": 1.0927553176879883,
+      "learning_rate": 0.00018441323494406095,
+      "loss": 2.132997512817383,
+      "step": 1650
+    },
+    {
+      "epoch": 0.23698204789607052,
+      "grad_norm": 1.3276400566101074,
+      "learning_rate": 0.00018431801951916213,
+      "loss": 2.1318714141845705,
+      "step": 1660
+    },
+    {
+      "epoch": 0.23840965059423963,
+      "grad_norm": 1.4104294776916504,
+      "learning_rate": 0.00018422280409426329,
+      "loss": 2.147476577758789,
+      "step": 1670
+    },
+    {
+      "epoch": 0.23983725329240874,
+      "grad_norm": 1.1079225540161133,
+      "learning_rate": 0.00018412758866936444,
+      "loss": 2.0742431640625,
+      "step": 1680
+    },
+    {
+      "epoch": 0.24126485599057781,
+      "grad_norm": 1.0819675922393799,
+      "learning_rate": 0.0001840323732444656,
+      "loss": 2.076985549926758,
+      "step": 1690
+    },
+    {
+      "epoch": 0.24269245868874692,
+      "grad_norm": 1.1977310180664062,
+      "learning_rate": 0.00018393715781956678,
+      "loss": 2.054951286315918,
+      "step": 1700
+    },
+    {
+      "epoch": 0.24412006138691603,
+      "grad_norm": 1.0316016674041748,
+      "learning_rate": 0.00018384194239466793,
+      "loss": 2.102836799621582,
+      "step": 1710
+    },
+    {
+      "epoch": 0.24554766408508513,
+      "grad_norm": 1.1931931972503662,
+      "learning_rate": 0.00018374672696976912,
+      "loss": 2.1211891174316406,
+      "step": 1720
+    },
+    {
+      "epoch": 0.2469752667832542,
+      "grad_norm": 1.2340854406356812,
+      "learning_rate": 0.00018365151154487027,
+      "loss": 2.2508804321289064,
+      "step": 1730
+    },
+    {
+      "epoch": 0.24840286948142332,
+      "grad_norm": 1.4340404272079468,
+      "learning_rate": 0.00018355629611997145,
+      "loss": 2.252923011779785,
+      "step": 1740
+    },
+    {
+      "epoch": 0.24983047217959242,
+      "grad_norm": 1.1949214935302734,
+      "learning_rate": 0.0001834610806950726,
+      "loss": 2.2155136108398437,
+      "step": 1750
+    },
+    {
+      "epoch": 0.25125807487776153,
+      "grad_norm": 1.0693628787994385,
+      "learning_rate": 0.0001833658652701738,
+      "loss": 2.0989776611328126,
+      "step": 1760
+    },
+    {
+      "epoch": 0.25268567757593063,
+      "grad_norm": 1.079628348350525,
+      "learning_rate": 0.00018327064984527492,
+      "loss": 2.235947036743164,
+      "step": 1770
+    },
+    {
+      "epoch": 0.25411328027409974,
+      "grad_norm": 1.0427007675170898,
+      "learning_rate": 0.0001831754344203761,
+      "loss": 2.2581958770751953,
+      "step": 1780
+    },
+    {
+      "epoch": 0.2555408829722688,
+      "grad_norm": 0.968690812587738,
+      "learning_rate": 0.00018308021899547726,
+      "loss": 2.152431869506836,
+      "step": 1790
+    },
+    {
+      "epoch": 0.2569684856704379,
+      "grad_norm": 1.4845409393310547,
+      "learning_rate": 0.00018298500357057844,
+      "loss": 2.082950401306152,
+      "step": 1800
+    },
+    {
+      "epoch": 0.258396088368607,
+      "grad_norm": 1.627851128578186,
+      "learning_rate": 0.0001828897881456796,
+      "loss": 2.1082151412963865,
+      "step": 1810
+    },
+    {
+      "epoch": 0.2598236910667761,
+      "grad_norm": 1.2222917079925537,
+      "learning_rate": 0.00018279457272078078,
+      "loss": 2.1010499954223634,
+      "step": 1820
+    },
+    {
+      "epoch": 0.2612512937649452,
+      "grad_norm": 1.1946543455123901,
+      "learning_rate": 0.00018269935729588193,
+      "loss": 2.20877685546875,
+      "step": 1830
+    },
+    {
+      "epoch": 0.2626788964631143,
+      "grad_norm": 1.2480523586273193,
+      "learning_rate": 0.00018260414187098312,
+      "loss": 2.1713356018066405,
+      "step": 1840
+    },
+    {
+      "epoch": 0.2641064991612834,
+      "grad_norm": 1.1162548065185547,
+      "learning_rate": 0.00018250892644608427,
+      "loss": 1.989072036743164,
+      "step": 1850
+    },
+    {
+      "epoch": 0.26553410185945253,
+      "grad_norm": 1.1317349672317505,
+      "learning_rate": 0.00018241371102118545,
+      "loss": 2.185866928100586,
+      "step": 1860
+    },
+    {
+      "epoch": 0.26696170455762164,
+      "grad_norm": 1.2101715803146362,
+      "learning_rate": 0.0001823184955962866,
+      "loss": 2.1214805603027345,
+      "step": 1870
+    },
+    {
+      "epoch": 0.2683893072557907,
+      "grad_norm": 1.036651611328125,
+      "learning_rate": 0.00018222328017138776,
+      "loss": 2.2269779205322267,
+      "step": 1880
+    },
+    {
+      "epoch": 0.2698169099539598,
+      "grad_norm": 1.2697064876556396,
+      "learning_rate": 0.00018212806474648895,
+      "loss": 2.188285064697266,
+      "step": 1890
+    },
+    {
+      "epoch": 0.2712445126521289,
+      "grad_norm": 1.086503028869629,
+      "learning_rate": 0.0001820328493215901,
+      "loss": 2.0915653228759767,
+      "step": 1900
+    },
+    {
+      "epoch": 0.272672115350298,
+      "grad_norm": 1.1498035192489624,
+      "learning_rate": 0.00018193763389669129,
+      "loss": 2.1859756469726563,
+      "step": 1910
+    },
+    {
+      "epoch": 0.2740997180484671,
+      "grad_norm": 1.1651625633239746,
+      "learning_rate": 0.00018184241847179244,
+      "loss": 2.1086774826049806,
+      "step": 1920
+    },
+    {
+      "epoch": 0.2755273207466362,
+      "grad_norm": 1.1872868537902832,
+      "learning_rate": 0.00018174720304689362,
+      "loss": 2.1062952041625977,
+      "step": 1930
+    },
+    {
+      "epoch": 0.2769549234448053,
+      "grad_norm": 1.0268173217773438,
+      "learning_rate": 0.00018165198762199478,
+      "loss": 2.056200408935547,
+      "step": 1940
+    },
+    {
+      "epoch": 0.27838252614297443,
+      "grad_norm": 1.0335297584533691,
+      "learning_rate": 0.00018155677219709596,
+      "loss": 2.17354793548584,
+      "step": 1950
+    },
+    {
+      "epoch": 0.27981012884114354,
+      "grad_norm": 1.1770780086517334,
+      "learning_rate": 0.0001814615567721971,
+      "loss": 2.123767852783203,
+      "step": 1960
+    },
+    {
+      "epoch": 0.2812377315393126,
+      "grad_norm": 1.1291216611862183,
+      "learning_rate": 0.00018136634134729827,
+      "loss": 2.1322879791259766,
+      "step": 1970
+    },
+    {
+      "epoch": 0.2826653342374817,
+      "grad_norm": 1.0811609029769897,
+      "learning_rate": 0.00018127112592239943,
+      "loss": 2.1403667449951174,
+      "step": 1980
+    },
+    {
+      "epoch": 0.2840929369356508,
+      "grad_norm": 1.3379665613174438,
+      "learning_rate": 0.0001811759104975006,
+      "loss": 2.227403450012207,
+      "step": 1990
+    },
+    {
+      "epoch": 0.2855205396338199,
+      "grad_norm": 1.2048321962356567,
+      "learning_rate": 0.00018108069507260176,
+      "loss": 2.320987319946289,
+      "step": 2000
+    },
+    {
+      "epoch": 0.286948142331989,
+      "grad_norm": 1.1182419061660767,
+      "learning_rate": 0.00018098547964770295,
+      "loss": 2.145694541931152,
+      "step": 2010
+    },
+    {
+      "epoch": 0.2883757450301581,
+      "grad_norm": 1.2120338678359985,
+      "learning_rate": 0.0001808902642228041,
+      "loss": 2.193251037597656,
+      "step": 2020
+    },
+    {
+      "epoch": 0.2898033477283272,
+      "grad_norm": 1.214379906654358,
+      "learning_rate": 0.00018079504879790528,
+      "loss": 2.183461570739746,
+      "step": 2030
+    },
+    {
+      "epoch": 0.2912309504264963,
+      "grad_norm": 1.2853455543518066,
+      "learning_rate": 0.00018069983337300644,
+      "loss": 2.105716323852539,
+      "step": 2040
+    },
+    {
+      "epoch": 0.29265855312466543,
+      "grad_norm": 0.9446287155151367,
+      "learning_rate": 0.00018060461794810762,
+      "loss": 2.0502334594726563,
+      "step": 2050
+    },
+    {
+      "epoch": 0.2940861558228345,
+      "grad_norm": 1.187010645866394,
+      "learning_rate": 0.00018050940252320875,
+      "loss": 2.0855840682983398,
+      "step": 2060
+    },
+    {
+      "epoch": 0.2955137585210036,
+      "grad_norm": 1.1547735929489136,
+      "learning_rate": 0.00018041418709830993,
+      "loss": 2.077458381652832,
+      "step": 2070
+    },
+    {
+      "epoch": 0.2969413612191727,
+      "grad_norm": 1.1305893659591675,
+      "learning_rate": 0.0001803189716734111,
+      "loss": 2.1422780990600585,
+      "step": 2080
+    },
+    {
+      "epoch": 0.2983689639173418,
+      "grad_norm": 1.2219164371490479,
+      "learning_rate": 0.00018022375624851227,
+      "loss": 2.1557044982910156,
+      "step": 2090
+    },
+    {
+      "epoch": 0.2997965666155109,
+      "grad_norm": 1.303414225578308,
+      "learning_rate": 0.00018012854082361343,
+      "loss": 2.0842567443847657,
+      "step": 2100
+    },
+    {
+      "epoch": 0.30122416931368,
+      "grad_norm": 1.1244888305664062,
+      "learning_rate": 0.0001800333253987146,
+      "loss": 2.0076160430908203,
+      "step": 2110
+    },
+    {
+      "epoch": 0.3026517720118491,
+      "grad_norm": 1.265690803527832,
+      "learning_rate": 0.00017993810997381576,
+      "loss": 2.2838394165039064,
+      "step": 2120
+    },
+    {
+      "epoch": 0.3040793747100182,
+      "grad_norm": 1.0629686117172241,
+      "learning_rate": 0.00017984289454891695,
+      "loss": 2.0467424392700195,
+      "step": 2130
+    },
+    {
+      "epoch": 0.3055069774081873,
+      "grad_norm": 1.1269965171813965,
+      "learning_rate": 0.0001797476791240181,
+      "loss": 2.059001159667969,
+      "step": 2140
+    },
+    {
+      "epoch": 0.3069345801063564,
+      "grad_norm": 1.0063384771347046,
+      "learning_rate": 0.00017965246369911926,
+      "loss": 2.054671859741211,
+      "step": 2150
+    },
+    {
+      "epoch": 0.3083621828045255,
+      "grad_norm": 1.696047306060791,
+      "learning_rate": 0.0001795572482742204,
+      "loss": 2.112416648864746,
+      "step": 2160
+    },
+    {
+      "epoch": 0.3097897855026946,
+      "grad_norm": 1.1593190431594849,
+      "learning_rate": 0.0001794620328493216,
+      "loss": 1.9681781768798827,
+      "step": 2170
+    },
+    {
+      "epoch": 0.3112173882008637,
+      "grad_norm": 1.1535143852233887,
+      "learning_rate": 0.00017936681742442275,
+      "loss": 1.9573915481567383,
+      "step": 2180
+    },
+    {
+      "epoch": 0.3126449908990328,
+      "grad_norm": 1.2937277555465698,
+      "learning_rate": 0.00017927160199952393,
+      "loss": 2.1334136962890624,
+      "step": 2190
+    },
+    {
+      "epoch": 0.3140725935972019,
+      "grad_norm": 1.1130894422531128,
+      "learning_rate": 0.0001791763865746251,
+      "loss": 2.054120635986328,
+      "step": 2200
+    },
+    {
+      "epoch": 0.315500196295371,
+      "grad_norm": 1.1255532503128052,
+      "learning_rate": 0.00017908117114972627,
+      "loss": 2.206426811218262,
+      "step": 2210
+    },
+    {
+      "epoch": 0.3169277989935401,
+      "grad_norm": 1.373878002166748,
+      "learning_rate": 0.00017898595572482743,
+      "loss": 2.016497802734375,
+      "step": 2220
+    },
+    {
+      "epoch": 0.31835540169170917,
+      "grad_norm": 0.9502859115600586,
+      "learning_rate": 0.0001788907402999286,
+      "loss": 2.1160964965820312,
+      "step": 2230
+    },
+    {
+      "epoch": 0.3197830043898783,
+      "grad_norm": 1.199417233467102,
+      "learning_rate": 0.00017879552487502976,
+      "loss": 2.219405746459961,
+      "step": 2240
+    },
+    {
+      "epoch": 0.3212106070880474,
+      "grad_norm": 1.2231884002685547,
+      "learning_rate": 0.00017870030945013092,
+      "loss": 2.1173473358154298,
+      "step": 2250
+    },
+    {
+      "epoch": 0.3226382097862165,
+      "grad_norm": 1.2296863794326782,
+      "learning_rate": 0.0001786050940252321,
+      "loss": 2.212135124206543,
+      "step": 2260
+    },
+    {
+      "epoch": 0.3240658124843856,
+      "grad_norm": 1.2749266624450684,
+      "learning_rate": 0.00017850987860033326,
+      "loss": 2.0644044876098633,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3254934151825547,
+      "grad_norm": 1.0321651697158813,
+      "learning_rate": 0.00017841466317543444,
+      "loss": 2.126047897338867,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3269210178807238,
+      "grad_norm": 1.131618857383728,
+      "learning_rate": 0.0001783194477505356,
+      "loss": 2.0301258087158205,
+      "step": 2290
+    },
+    {
+      "epoch": 0.3283486205788929,
+      "grad_norm": 0.9653699994087219,
+      "learning_rate": 0.00017822423232563678,
+      "loss": 2.172132873535156,
+      "step": 2300
+    },
+    {
+      "epoch": 0.329776223277062,
+      "grad_norm": 1.1435012817382812,
+      "learning_rate": 0.00017812901690073793,
+      "loss": 2.1726940155029295,
+      "step": 2310
+    },
+    {
+      "epoch": 0.33120382597523107,
+      "grad_norm": 1.3378164768218994,
+      "learning_rate": 0.00017803380147583911,
+      "loss": 2.1459815979003904,
+      "step": 2320
+    },
+    {
+      "epoch": 0.3326314286734002,
+      "grad_norm": 1.0936335325241089,
+      "learning_rate": 0.00017793858605094027,
+      "loss": 2.11383113861084,
+      "step": 2330
+    },
+    {
+      "epoch": 0.3340590313715693,
+      "grad_norm": 1.7248929738998413,
+      "learning_rate": 0.00017784337062604142,
+      "loss": 2.060173225402832,
+      "step": 2340
+    },
+    {
+      "epoch": 0.3354866340697384,
+      "grad_norm": 1.4490845203399658,
+      "learning_rate": 0.00017774815520114258,
+      "loss": 2.168626403808594,
+      "step": 2350
+    },
+    {
+      "epoch": 0.3369142367679075,
+      "grad_norm": 1.4273464679718018,
+      "learning_rate": 0.00017765293977624376,
+      "loss": 2.1859357833862303,
+      "step": 2360
+    },
+    {
+      "epoch": 0.3383418394660766,
+      "grad_norm": 1.1335994005203247,
+      "learning_rate": 0.00017755772435134492,
+      "loss": 2.2323787689208983,
+      "step": 2370
+    },
+    {
+      "epoch": 0.3397694421642457,
+      "grad_norm": 1.0822454690933228,
+      "learning_rate": 0.0001774625089264461,
+      "loss": 2.1954828262329102,
+      "step": 2380
+    },
+    {
+      "epoch": 0.3411970448624148,
+      "grad_norm": 1.3556091785430908,
+      "learning_rate": 0.00017736729350154726,
+      "loss": 2.086533546447754,
+      "step": 2390
+    },
+    {
+      "epoch": 0.3426246475605839,
+      "grad_norm": 1.0348728895187378,
+      "learning_rate": 0.00017727207807664844,
+      "loss": 2.063125419616699,
+      "step": 2400
+    },
+    {
+      "epoch": 0.34405225025875297,
+      "grad_norm": 0.9327099323272705,
+      "learning_rate": 0.0001771768626517496,
+      "loss": 2.2520879745483398,
+      "step": 2410
+    },
+    {
+      "epoch": 0.3454798529569221,
+      "grad_norm": 1.161893367767334,
+      "learning_rate": 0.00017708164722685078,
+      "loss": 1.9637351989746095,
+      "step": 2420
+    },
+    {
+      "epoch": 0.3469074556550912,
+      "grad_norm": 1.0252056121826172,
+      "learning_rate": 0.00017698643180195193,
+      "loss": 2.105502700805664,
+      "step": 2430
+    },
+    {
+      "epoch": 0.3483350583532603,
+      "grad_norm": 1.0173977613449097,
+      "learning_rate": 0.00017689121637705309,
+      "loss": 2.1620779037475586,
+      "step": 2440
+    },
+    {
+      "epoch": 0.3497626610514294,
+      "grad_norm": 1.0667582750320435,
+      "learning_rate": 0.00017679600095215424,
+      "loss": 2.1498435974121093,
+      "step": 2450
+    },
+    {
+      "epoch": 0.3511902637495985,
+      "grad_norm": 1.1689473390579224,
+      "learning_rate": 0.00017670078552725542,
+      "loss": 2.010816192626953,
+      "step": 2460
+    },
+    {
+      "epoch": 0.3526178664477676,
+      "grad_norm": 1.1713252067565918,
+      "learning_rate": 0.00017660557010235658,
+      "loss": 2.1770673751831056,
+      "step": 2470
+    },
+    {
+      "epoch": 0.3540454691459367,
+      "grad_norm": 1.1741546392440796,
+      "learning_rate": 0.00017651035467745776,
+      "loss": 2.2705522537231446,
+      "step": 2480
+    },
+    {
+      "epoch": 0.35547307184410576,
+      "grad_norm": 1.1165494918823242,
+      "learning_rate": 0.00017641513925255892,
+      "loss": 2.2441898345947267,
+      "step": 2490
+    },
+    {
+      "epoch": 0.35690067454227486,
+      "grad_norm": 1.128848910331726,
+      "learning_rate": 0.0001763199238276601,
+      "loss": 1.966132354736328,
+      "step": 2500
+    },
+    {
+      "epoch": 0.35832827724044397,
+      "grad_norm": 1.1416059732437134,
+      "learning_rate": 0.00017622470840276125,
+      "loss": 2.129666328430176,
+      "step": 2510
+    },
+    {
+      "epoch": 0.3597558799386131,
+      "grad_norm": 1.3967465162277222,
+      "learning_rate": 0.00017612949297786244,
+      "loss": 2.068239212036133,
+      "step": 2520
+    },
+    {
+      "epoch": 0.3611834826367822,
+      "grad_norm": 1.1982859373092651,
+      "learning_rate": 0.00017603427755296357,
+      "loss": 2.0356422424316407,
+      "step": 2530
+    },
+    {
+      "epoch": 0.3626110853349513,
+      "grad_norm": 1.2561085224151611,
+      "learning_rate": 0.00017593906212806475,
+      "loss": 2.1692792892456056,
+      "step": 2540
+    },
+    {
+      "epoch": 0.3640386880331204,
+      "grad_norm": 1.2402371168136597,
+      "learning_rate": 0.0001758438467031659,
+      "loss": 2.018773078918457,
+      "step": 2550
+    },
+    {
+      "epoch": 0.3654662907312895,
+      "grad_norm": 1.0577623844146729,
+      "learning_rate": 0.00017574863127826709,
+      "loss": 2.2117090225219727,
+      "step": 2560
+    },
+    {
+      "epoch": 0.3668938934294586,
+      "grad_norm": 1.3217538595199585,
+      "learning_rate": 0.00017565341585336824,
+      "loss": 2.19326229095459,
+      "step": 2570
+    },
+    {
+      "epoch": 0.36832149612762766,
+      "grad_norm": 1.2093907594680786,
+      "learning_rate": 0.00017555820042846942,
+      "loss": 2.072580909729004,
+      "step": 2580
+    },
+    {
+      "epoch": 0.36974909882579676,
+      "grad_norm": 1.1893237829208374,
+      "learning_rate": 0.00017546298500357058,
+      "loss": 2.0644367218017576,
+      "step": 2590
+    },
+    {
+      "epoch": 0.37117670152396587,
+      "grad_norm": 1.116391658782959,
+      "learning_rate": 0.00017536776957867176,
+      "loss": 2.0989030838012694,
+      "step": 2600
+    },
+    {
+      "epoch": 0.372604304222135,
+      "grad_norm": 1.1465907096862793,
+      "learning_rate": 0.00017527255415377292,
+      "loss": 2.22904109954834,
+      "step": 2610
+    },
+    {
+      "epoch": 0.3740319069203041,
+      "grad_norm": 1.2656993865966797,
+      "learning_rate": 0.0001751773387288741,
+      "loss": 2.162650489807129,
+      "step": 2620
+    },
+    {
+      "epoch": 0.3754595096184732,
+      "grad_norm": 1.0960391759872437,
+      "learning_rate": 0.00017508212330397525,
+      "loss": 2.0461801528930663,
+      "step": 2630
+    },
+    {
+      "epoch": 0.3768871123166423,
+      "grad_norm": 1.1352872848510742,
+      "learning_rate": 0.0001749869078790764,
+      "loss": 2.142223930358887,
+      "step": 2640
+    },
+    {
+      "epoch": 0.3783147150148114,
+      "grad_norm": 1.3211382627487183,
+      "learning_rate": 0.0001748916924541776,
+      "loss": 2.086302375793457,
+      "step": 2650
+    },
+    {
+      "epoch": 0.3797423177129805,
+      "grad_norm": 1.3286374807357788,
+      "learning_rate": 0.00017479647702927875,
+      "loss": 2.1871686935424806,
+      "step": 2660
+    },
+    {
+      "epoch": 0.38116992041114955,
+      "grad_norm": 1.1504572629928589,
+      "learning_rate": 0.00017470126160437993,
+      "loss": 2.2135593414306642,
+      "step": 2670
+    },
+    {
+      "epoch": 0.38259752310931866,
+      "grad_norm": 1.1061992645263672,
+      "learning_rate": 0.00017460604617948108,
+      "loss": 1.9979249954223632,
+      "step": 2680
+    },
+    {
+      "epoch": 0.38402512580748777,
+      "grad_norm": 1.21872079372406,
+      "learning_rate": 0.00017451083075458227,
+      "loss": 2.1905019760131834,
+      "step": 2690
+    },
+    {
+      "epoch": 0.38545272850565687,
+      "grad_norm": 0.9169778823852539,
+      "learning_rate": 0.00017441561532968342,
+      "loss": 2.096824836730957,
+      "step": 2700
+    },
+    {
+      "epoch": 0.386880331203826,
+      "grad_norm": 1.1782190799713135,
+      "learning_rate": 0.0001743203999047846,
+      "loss": 2.1858846664428713,
+      "step": 2710
+    },
+    {
+      "epoch": 0.3883079339019951,
+      "grad_norm": 1.0241059064865112,
+      "learning_rate": 0.00017422518447988573,
+      "loss": 2.1724557876586914,
+      "step": 2720
+    },
+    {
+      "epoch": 0.3897355366001642,
+      "grad_norm": 1.3461824655532837,
+      "learning_rate": 0.00017412996905498692,
+      "loss": 2.0981250762939454,
+      "step": 2730
+    },
+    {
+      "epoch": 0.3911631392983333,
+      "grad_norm": 1.231691837310791,
+      "learning_rate": 0.00017403475363008807,
+      "loss": 2.000545692443848,
+      "step": 2740
+    },
+    {
+      "epoch": 0.39259074199650235,
+      "grad_norm": 1.1789937019348145,
+      "learning_rate": 0.00017393953820518925,
+      "loss": 1.9449064254760742,
+      "step": 2750
+    },
+    {
+      "epoch": 0.39401834469467145,
+      "grad_norm": 1.119533896446228,
+      "learning_rate": 0.0001738443227802904,
+      "loss": 2.0872426986694337,
+      "step": 2760
+    },
+    {
+      "epoch": 0.39544594739284056,
+      "grad_norm": 1.034921407699585,
+      "learning_rate": 0.0001737491073553916,
+      "loss": 2.0502944946289063,
+      "step": 2770
+    },
+    {
+      "epoch": 0.39687355009100966,
+      "grad_norm": 1.1675975322723389,
+      "learning_rate": 0.00017365389193049275,
+      "loss": 2.0751968383789063,
+      "step": 2780
+    },
+    {
+      "epoch": 0.39830115278917877,
+      "grad_norm": 0.9863481521606445,
+      "learning_rate": 0.00017355867650559393,
+      "loss": 2.216383171081543,
+      "step": 2790
+    },
+    {
+      "epoch": 0.3997287554873479,
+      "grad_norm": 1.0700448751449585,
+      "learning_rate": 0.00017346346108069508,
+      "loss": 2.037568473815918,
+      "step": 2800
+    },
+    {
+      "epoch": 0.401156358185517,
+      "grad_norm": 1.5234819650650024,
+      "learning_rate": 0.00017336824565579627,
+      "loss": 2.135871696472168,
+      "step": 2810
+    },
+    {
+      "epoch": 0.4025839608836861,
+      "grad_norm": 1.312696933746338,
+      "learning_rate": 0.0001732730302308974,
+      "loss": 2.123103713989258,
+      "step": 2820
+    },
+    {
+      "epoch": 0.4040115635818552,
+      "grad_norm": 1.4110193252563477,
+      "learning_rate": 0.00017317781480599858,
+      "loss": 2.2066755294799805,
+      "step": 2830
+    },
+    {
+      "epoch": 0.40543916628002424,
+      "grad_norm": 1.1026965379714966,
+      "learning_rate": 0.00017308259938109973,
+      "loss": 2.026701736450195,
+      "step": 2840
+    },
+    {
+      "epoch": 0.40686676897819335,
+      "grad_norm": 1.1809606552124023,
+      "learning_rate": 0.00017298738395620092,
+      "loss": 2.013088035583496,
+      "step": 2850
+    },
+    {
+      "epoch": 0.40829437167636246,
+      "grad_norm": 1.281699538230896,
+      "learning_rate": 0.00017289216853130207,
+      "loss": 2.212848663330078,
+      "step": 2860
+    },
+    {
+      "epoch": 0.40972197437453156,
+      "grad_norm": 1.4359161853790283,
+      "learning_rate": 0.00017279695310640325,
+      "loss": 2.3273263931274415,
+      "step": 2870
+    },
+    {
+      "epoch": 0.41114957707270067,
+      "grad_norm": 1.1231915950775146,
+      "learning_rate": 0.0001727017376815044,
+      "loss": 2.024250602722168,
+      "step": 2880
+    },
+    {
+      "epoch": 0.4125771797708698,
+      "grad_norm": 1.1361498832702637,
+      "learning_rate": 0.0001726065222566056,
+      "loss": 2.0198749542236327,
+      "step": 2890
+    },
+    {
+      "epoch": 0.4140047824690389,
+      "grad_norm": 1.277862310409546,
+      "learning_rate": 0.00017251130683170675,
+      "loss": 2.0423700332641603,
+      "step": 2900
+    },
+    {
+      "epoch": 0.415432385167208,
+      "grad_norm": 1.3887676000595093,
+      "learning_rate": 0.0001724160914068079,
+      "loss": 2.1957927703857423,
+      "step": 2910
+    },
+    {
+      "epoch": 0.4168599878653771,
+      "grad_norm": 1.102116584777832,
+      "learning_rate": 0.00017232087598190906,
+      "loss": 1.9576358795166016,
+      "step": 2920
+    },
+    {
+      "epoch": 0.41828759056354614,
+      "grad_norm": 1.658210277557373,
+      "learning_rate": 0.00017222566055701024,
+      "loss": 2.1199745178222655,
+      "step": 2930
+    },
+    {
+      "epoch": 0.41971519326171525,
+      "grad_norm": 1.3684505224227905,
+      "learning_rate": 0.0001721304451321114,
+      "loss": 2.1091520309448244,
+      "step": 2940
+    },
+    {
+      "epoch": 0.42114279595988435,
+      "grad_norm": 1.1813336610794067,
+      "learning_rate": 0.00017203522970721258,
+      "loss": 2.0536418914794923,
+      "step": 2950
+    },
+    {
+      "epoch": 0.42257039865805346,
+      "grad_norm": 1.2007261514663696,
+      "learning_rate": 0.00017194001428231373,
+      "loss": 2.053822708129883,
+      "step": 2960
+    },
+    {
+      "epoch": 0.42399800135622256,
+      "grad_norm": 1.0350509881973267,
+      "learning_rate": 0.00017184479885741491,
+      "loss": 1.9133419036865233,
+      "step": 2970
+    },
+    {
+      "epoch": 0.42542560405439167,
+      "grad_norm": 1.0523207187652588,
+      "learning_rate": 0.00017174958343251607,
+      "loss": 2.0608654022216797,
+      "step": 2980
+    },
+    {
+      "epoch": 0.4268532067525608,
+      "grad_norm": 0.952148973941803,
+      "learning_rate": 0.00017165436800761725,
+      "loss": 2.1483791351318358,
+      "step": 2990
+    },
+    {
+      "epoch": 0.4282808094507299,
+      "grad_norm": 1.0913255214691162,
+      "learning_rate": 0.0001715591525827184,
+      "loss": 2.155402946472168,
+      "step": 3000
+    },
+    {
+      "epoch": 0.429708412148899,
+      "grad_norm": 1.05339777469635,
+      "learning_rate": 0.00017146393715781956,
+      "loss": 2.1396499633789063,
+      "step": 3010
+    },
+    {
+      "epoch": 0.43113601484706804,
+      "grad_norm": 1.2329785823822021,
+      "learning_rate": 0.00017136872173292075,
+      "loss": 2.2712575912475588,
+      "step": 3020
+    },
+    {
+      "epoch": 0.43256361754523714,
+      "grad_norm": 1.1520178318023682,
+      "learning_rate": 0.0001712735063080219,
+      "loss": 2.0944181442260743,
+      "step": 3030
+    },
+    {
+      "epoch": 0.43399122024340625,
+      "grad_norm": 1.1466962099075317,
+      "learning_rate": 0.00017117829088312308,
+      "loss": 2.064352607727051,
+      "step": 3040
+    },
+    {
+      "epoch": 0.43541882294157536,
+      "grad_norm": 1.2289594411849976,
+      "learning_rate": 0.00017108307545822424,
+      "loss": 2.2511341094970705,
+      "step": 3050
+    },
+    {
+      "epoch": 0.43684642563974446,
+      "grad_norm": 1.0378469228744507,
+      "learning_rate": 0.00017098786003332542,
+      "loss": 2.0552734375,
+      "step": 3060
+    },
+    {
+      "epoch": 0.43827402833791357,
+      "grad_norm": 1.1334617137908936,
+      "learning_rate": 0.00017089264460842658,
+      "loss": 2.1504894256591798,
+      "step": 3070
+    },
+    {
+      "epoch": 0.4397016310360827,
+      "grad_norm": 1.2314560413360596,
+      "learning_rate": 0.00017079742918352776,
+      "loss": 2.1562742233276366,
+      "step": 3080
+    },
+    {
+      "epoch": 0.4411292337342518,
+      "grad_norm": 1.1508898735046387,
+      "learning_rate": 0.00017070221375862891,
+      "loss": 2.0091711044311524,
+      "step": 3090
+    },
+    {
+      "epoch": 0.44255683643242083,
+      "grad_norm": 1.7214949131011963,
+      "learning_rate": 0.00017060699833373007,
+      "loss": 2.084541893005371,
+      "step": 3100
+    },
+    {
+      "epoch": 0.44398443913058994,
+      "grad_norm": 1.452836036682129,
+      "learning_rate": 0.00017051178290883122,
+      "loss": 2.167698097229004,
+      "step": 3110
+    },
+    {
+      "epoch": 0.44541204182875904,
+      "grad_norm": 1.1506155729293823,
+      "learning_rate": 0.0001704165674839324,
+      "loss": 2.1772172927856444,
+      "step": 3120
+    },
+    {
+      "epoch": 0.44683964452692815,
+      "grad_norm": 1.6658185720443726,
+      "learning_rate": 0.00017032135205903356,
+      "loss": 2.1950592041015624,
+      "step": 3130
+    },
+    {
+      "epoch": 0.44826724722509725,
+      "grad_norm": 1.0545748472213745,
+      "learning_rate": 0.00017022613663413474,
+      "loss": 2.052364730834961,
+      "step": 3140
+    },
+    {
+      "epoch": 0.44969484992326636,
+      "grad_norm": 1.3302888870239258,
+      "learning_rate": 0.0001701309212092359,
+      "loss": 2.2422548294067384,
+      "step": 3150
+    },
+    {
+      "epoch": 0.45112245262143547,
+      "grad_norm": 1.4070645570755005,
+      "learning_rate": 0.00017003570578433708,
+      "loss": 2.1141366958618164,
+      "step": 3160
+    },
+    {
+      "epoch": 0.45255005531960457,
+      "grad_norm": 1.0993238687515259,
+      "learning_rate": 0.00016994049035943824,
+      "loss": 2.1006706237792967,
+      "step": 3170
+    },
+    {
+      "epoch": 0.4539776580177737,
+      "grad_norm": 1.1617511510849,
+      "learning_rate": 0.00016984527493453942,
+      "loss": 1.9723478317260743,
+      "step": 3180
+    },
+    {
+      "epoch": 0.45540526071594273,
+      "grad_norm": 1.5463227033615112,
+      "learning_rate": 0.00016975005950964058,
+      "loss": 2.104926109313965,
+      "step": 3190
+    },
+    {
+      "epoch": 0.45683286341411183,
+      "grad_norm": 1.2094980478286743,
+      "learning_rate": 0.00016965484408474173,
+      "loss": 1.9171577453613282,
+      "step": 3200
+    },
+    {
+      "epoch": 0.45826046611228094,
+      "grad_norm": 1.103075385093689,
+      "learning_rate": 0.00016955962865984289,
+      "loss": 2.1229742050170897,
+      "step": 3210
+    },
+    {
+      "epoch": 0.45968806881045005,
+      "grad_norm": 1.3477221727371216,
+      "learning_rate": 0.00016946441323494407,
+      "loss": 2.0110040664672852,
+      "step": 3220
+    },
+    {
+      "epoch": 0.46111567150861915,
+      "grad_norm": 1.1711978912353516,
+      "learning_rate": 0.00016936919781004522,
+      "loss": 2.1763769149780274,
+      "step": 3230
+    },
+    {
+      "epoch": 0.46254327420678826,
+      "grad_norm": 1.1173983812332153,
+      "learning_rate": 0.0001692739823851464,
+      "loss": 2.0923931121826174,
+      "step": 3240
+    },
+    {
+      "epoch": 0.46397087690495736,
+      "grad_norm": 1.3516672849655151,
+      "learning_rate": 0.00016917876696024756,
+      "loss": 2.1028839111328126,
+      "step": 3250
+    },
+    {
+      "epoch": 0.46539847960312647,
+      "grad_norm": 1.187572717666626,
+      "learning_rate": 0.00016908355153534874,
+      "loss": 2.039937973022461,
+      "step": 3260
+    },
+    {
+      "epoch": 0.4668260823012956,
+      "grad_norm": 1.3026293516159058,
+      "learning_rate": 0.0001689883361104499,
+      "loss": 2.0570196151733398,
+      "step": 3270
+    },
+    {
+      "epoch": 0.4682536849994646,
+      "grad_norm": 1.1978344917297363,
+      "learning_rate": 0.00016889312068555108,
+      "loss": 2.077353858947754,
+      "step": 3280
+    },
+    {
+      "epoch": 0.46968128769763373,
+      "grad_norm": 1.0989017486572266,
+      "learning_rate": 0.0001687979052606522,
+      "loss": 1.9933378219604492,
+      "step": 3290
+    },
+    {
+      "epoch": 0.47110889039580284,
+      "grad_norm": 1.1777777671813965,
+      "learning_rate": 0.0001687026898357534,
+      "loss": 2.197841262817383,
+      "step": 3300
+    },
+    {
+      "epoch": 0.47253649309397194,
+      "grad_norm": 1.8024139404296875,
+      "learning_rate": 0.00016860747441085455,
+      "loss": 2.1699588775634764,
+      "step": 3310
+    },
+    {
+      "epoch": 0.47396409579214105,
+      "grad_norm": 1.0226041078567505,
+      "learning_rate": 0.00016851225898595573,
+      "loss": 2.0379987716674806,
+      "step": 3320
+    },
+    {
+      "epoch": 0.47539169849031016,
+      "grad_norm": 2.29010009765625,
+      "learning_rate": 0.00016841704356105689,
+      "loss": 2.08388614654541,
+      "step": 3330
+    },
+    {
+      "epoch": 0.47681930118847926,
+      "grad_norm": 0.9974448084831238,
+      "learning_rate": 0.00016832182813615807,
+      "loss": 2.0867422103881834,
+      "step": 3340
+    },
+    {
+      "epoch": 0.47824690388664837,
+      "grad_norm": 1.1733876466751099,
+      "learning_rate": 0.00016822661271125922,
+      "loss": 2.038913917541504,
+      "step": 3350
+    },
+    {
+      "epoch": 0.4796745065848175,
+      "grad_norm": 1.0234184265136719,
+      "learning_rate": 0.0001681313972863604,
+      "loss": 2.1241840362548827,
+      "step": 3360
+    },
+    {
+      "epoch": 0.4811021092829865,
+      "grad_norm": 1.4544063806533813,
+      "learning_rate": 0.00016803618186146156,
+      "loss": 2.044015121459961,
+      "step": 3370
+    },
+    {
+      "epoch": 0.48252971198115563,
+      "grad_norm": 1.368973731994629,
+      "learning_rate": 0.00016794096643656274,
+      "loss": 2.019906425476074,
+      "step": 3380
+    },
+    {
+      "epoch": 0.48395731467932473,
+      "grad_norm": 1.168421745300293,
+      "learning_rate": 0.0001678457510116639,
+      "loss": 2.0222070693969725,
+      "step": 3390
+    },
+    {
+      "epoch": 0.48538491737749384,
+      "grad_norm": 1.2671949863433838,
+      "learning_rate": 0.00016775053558676505,
+      "loss": 2.1466264724731445,
+      "step": 3400
+    },
+    {
+      "epoch": 0.48681252007566295,
+      "grad_norm": 1.0771664381027222,
+      "learning_rate": 0.00016765532016186624,
+      "loss": 2.0024179458618163,
+      "step": 3410
+    },
+    {
+      "epoch": 0.48824012277383205,
+      "grad_norm": 1.2461390495300293,
+      "learning_rate": 0.0001675601047369674,
+      "loss": 2.0828195571899415,
+      "step": 3420
+    },
+    {
+      "epoch": 0.48966772547200116,
+      "grad_norm": 1.6071606874465942,
+      "learning_rate": 0.00016746488931206857,
+      "loss": 1.9849077224731446,
+      "step": 3430
+    },
+    {
+      "epoch": 0.49109532817017026,
+      "grad_norm": 1.0467002391815186,
+      "learning_rate": 0.00016736967388716973,
+      "loss": 2.0019205093383787,
+      "step": 3440
+    },
+    {
+      "epoch": 0.4925229308683393,
+      "grad_norm": 1.0588996410369873,
+      "learning_rate": 0.0001672744584622709,
+      "loss": 2.0819076538085937,
+      "step": 3450
+    },
+    {
+      "epoch": 0.4939505335665084,
+      "grad_norm": 0.9776011109352112,
+      "learning_rate": 0.00016717924303737207,
+      "loss": 1.9890733718872071,
+      "step": 3460
+    },
+    {
+      "epoch": 0.4953781362646775,
+      "grad_norm": 1.439799427986145,
+      "learning_rate": 0.00016708402761247325,
+      "loss": 2.1648094177246096,
+      "step": 3470
+    },
+    {
+      "epoch": 0.49680573896284663,
+      "grad_norm": 1.063666820526123,
+      "learning_rate": 0.0001669888121875744,
+      "loss": 1.9934938430786133,
+      "step": 3480
+    },
+    {
+      "epoch": 0.49823334166101574,
+      "grad_norm": 1.0971171855926514,
+      "learning_rate": 0.00016689359676267556,
+      "loss": 2.0755395889282227,
+      "step": 3490
+    },
+    {
+      "epoch": 0.49966094435918484,
+      "grad_norm": 1.3422017097473145,
+      "learning_rate": 0.00016679838133777672,
+      "loss": 2.080609130859375,
+      "step": 3500
+    },
+    {
+      "epoch": 0.501088547057354,
+      "grad_norm": 1.3471558094024658,
+      "learning_rate": 0.0001667031659128779,
+      "loss": 2.0093482971191405,
+      "step": 3510
+    },
+    {
+      "epoch": 0.5025161497555231,
+      "grad_norm": 1.0592118501663208,
+      "learning_rate": 0.00016660795048797905,
+      "loss": 1.997859573364258,
+      "step": 3520
+    },
+    {
+      "epoch": 0.5039437524536922,
+      "grad_norm": 1.2080049514770508,
+      "learning_rate": 0.00016651273506308024,
+      "loss": 2.133045959472656,
+      "step": 3530
+    },
+    {
+      "epoch": 0.5053713551518613,
+      "grad_norm": 0.9516943097114563,
+      "learning_rate": 0.0001664175196381814,
+      "loss": 2.04406795501709,
+      "step": 3540
+    },
+    {
+      "epoch": 0.5067989578500304,
+      "grad_norm": 0.9474464058876038,
+      "learning_rate": 0.00016632230421328257,
+      "loss": 2.161180877685547,
+      "step": 3550
+    },
+    {
+      "epoch": 0.5082265605481995,
+      "grad_norm": 1.1096711158752441,
+      "learning_rate": 0.00016622708878838373,
+      "loss": 2.1555700302124023,
+      "step": 3560
+    },
+    {
+      "epoch": 0.5096541632463686,
+      "grad_norm": 1.3584140539169312,
+      "learning_rate": 0.0001661318733634849,
+      "loss": 2.018060302734375,
+      "step": 3570
+    },
+    {
+      "epoch": 0.5110817659445376,
+      "grad_norm": 1.6246408224105835,
+      "learning_rate": 0.00016603665793858604,
+      "loss": 2.2220287322998047,
+      "step": 3580
+    },
+    {
+      "epoch": 0.5125093686427067,
+      "grad_norm": 1.2138937711715698,
+      "learning_rate": 0.00016594144251368722,
+      "loss": 2.069137954711914,
+      "step": 3590
+    },
+    {
+      "epoch": 0.5139369713408758,
+      "grad_norm": 1.2093039751052856,
+      "learning_rate": 0.00016584622708878838,
+      "loss": 2.109259033203125,
+      "step": 3600
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 21015,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.5048344093257728e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/checkpoint-3600/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e15cb1849f07f1496673bd566c14ee0b735bc8db1b29b7446b0bc032f6b064
+size 5841