Upload task output zeus-ld-intercode-1780902488

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: None
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/cache/models/Jordansky--augmented-f560e4e6ee71e78d
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff70e1d9a8265b115e8032d2a8a6f417ebb39a6d538bc746ea39410afc4ab82f
+size 1057033224

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

loss.txt

@@ -0,0 +1 @@
+761,no_eval

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1402 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.28164322723908214,
+  "eval_steps": 500,
+  "global_step": 761,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001850481125092524,
+      "grad_norm": 12.649392127990723,
+      "learning_rate": 6.325018450184502e-07,
+      "loss": 1.3603,
+      "mean_token_accuracy": 0.7554670929908752,
+      "num_tokens": 185821.0,
+      "step": 5
+    },
+    {
+      "epoch": 0.003700962250185048,
+      "grad_norm": 9.217720985412598,
+      "learning_rate": 1.4231291512915131e-06,
+      "loss": 1.2892,
+      "mean_token_accuracy": 0.7647651076316834,
+      "num_tokens": 374500.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.0055514433752775726,
+      "grad_norm": 4.445683002471924,
+      "learning_rate": 2.2137564575645756e-06,
+      "loss": 1.1434,
+      "mean_token_accuracy": 0.7748205065727234,
+      "num_tokens": 563560.0,
+      "step": 15
+    },
+    {
+      "epoch": 0.007401924500370096,
+      "grad_norm": 3.0582313537597656,
+      "learning_rate": 3.004383763837638e-06,
+      "loss": 0.9881,
+      "mean_token_accuracy": 0.7829701185226441,
+      "num_tokens": 746605.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.009252405625462621,
+      "grad_norm": 2.2693915367126465,
+      "learning_rate": 3.795011070110701e-06,
+      "loss": 0.8071,
+      "mean_token_accuracy": 0.8049428701400757,
+      "num_tokens": 932791.0,
+      "step": 25
+    },
+    {
+      "epoch": 0.011102886750555145,
+      "grad_norm": 2.1072044372558594,
+      "learning_rate": 4.585638376383764e-06,
+      "loss": 0.6524,
+      "mean_token_accuracy": 0.8388792037963867,
+      "num_tokens": 1117440.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.012953367875647668,
+      "grad_norm": 1.808276891708374,
+      "learning_rate": 5.376265682656827e-06,
+      "loss": 0.4326,
+      "mean_token_accuracy": 0.8950777530670166,
+      "num_tokens": 1304726.0,
+      "step": 35
+    },
+    {
+      "epoch": 0.014803849000740192,
+      "grad_norm": 1.2398462295532227,
+      "learning_rate": 6.1668929889298895e-06,
+      "loss": 0.2749,
+      "mean_token_accuracy": 0.9303830146789551,
+      "num_tokens": 1495300.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.016654330125832718,
+      "grad_norm": 0.8731722235679626,
+      "learning_rate": 6.957520295202952e-06,
+      "loss": 0.1441,
+      "mean_token_accuracy": 0.9656038403511047,
+      "num_tokens": 1685752.0,
+      "step": 45
+    },
+    {
+      "epoch": 0.018504811250925242,
+      "grad_norm": 0.5143530964851379,
+      "learning_rate": 7.748147601476015e-06,
+      "loss": 0.1277,
+      "mean_token_accuracy": 0.966814661026001,
+      "num_tokens": 1868182.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.020355292376017766,
+      "grad_norm": 0.4487469792366028,
+      "learning_rate": 8.538774907749078e-06,
+      "loss": 0.0714,
+      "mean_token_accuracy": 0.9793548464775086,
+      "num_tokens": 2057318.0,
+      "step": 55
+    },
+    {
+      "epoch": 0.02220577350111029,
+      "grad_norm": 0.35000377893447876,
+      "learning_rate": 9.32940221402214e-06,
+      "loss": 0.0824,
+      "mean_token_accuracy": 0.9777526021003723,
+      "num_tokens": 2245866.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.024056254626202814,
+      "grad_norm": 0.1533830463886261,
+      "learning_rate": 1.0120029520295203e-05,
+      "loss": 0.0538,
+      "mean_token_accuracy": 0.9833965539932251,
+      "num_tokens": 2433677.0,
+      "step": 65
+    },
+    {
+      "epoch": 0.025906735751295335,
+      "grad_norm": 0.387611985206604,
+      "learning_rate": 1.0910656826568265e-05,
+      "loss": 0.044,
+      "mean_token_accuracy": 0.9864847898483277,
+      "num_tokens": 2623909.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.02775721687638786,
+      "grad_norm": 0.19073283672332764,
+      "learning_rate": 1.170128413284133e-05,
+      "loss": 0.0295,
+      "mean_token_accuracy": 0.9899263978004456,
+      "num_tokens": 2802983.0,
+      "step": 75
+    },
+    {
+      "epoch": 0.029607698001480384,
+      "grad_norm": 0.7775747179985046,
+      "learning_rate": 1.2491911439114392e-05,
+      "loss": 0.0263,
+      "mean_token_accuracy": 0.9908901691436768,
+      "num_tokens": 2986382.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.03145817912657291,
+      "grad_norm": 0.4266139566898346,
+      "learning_rate": 1.3282538745387456e-05,
+      "loss": 0.0285,
+      "mean_token_accuracy": 0.9905906915664673,
+      "num_tokens": 3169983.0,
+      "step": 85
+    },
+    {
+      "epoch": 0.033308660251665435,
+      "grad_norm": 0.10814467072486877,
+      "learning_rate": 1.4073166051660517e-05,
+      "loss": 0.0204,
+      "mean_token_accuracy": 0.9922121882438659,
+      "num_tokens": 3357363.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.035159141376757956,
+      "grad_norm": 0.354408323764801,
+      "learning_rate": 1.4863793357933581e-05,
+      "loss": 0.0223,
+      "mean_token_accuracy": 0.9915817618370056,
+      "num_tokens": 3546512.0,
+      "step": 95
+    },
+    {
+      "epoch": 0.037009622501850484,
+      "grad_norm": 0.1480749547481537,
+      "learning_rate": 1.5654420664206644e-05,
+      "loss": 0.0185,
+      "mean_token_accuracy": 0.992724347114563,
+      "num_tokens": 3735563.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.038860103626943004,
+      "grad_norm": 0.1472485065460205,
+      "learning_rate": 1.6445047970479706e-05,
+      "loss": 0.0183,
+      "mean_token_accuracy": 0.992663311958313,
+      "num_tokens": 3922268.0,
+      "step": 105
+    },
+    {
+      "epoch": 0.04071058475203553,
+      "grad_norm": 0.10689185559749603,
+      "learning_rate": 1.723567527675277e-05,
+      "loss": 0.0188,
+      "mean_token_accuracy": 0.9926952004432679,
+      "num_tokens": 4110402.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.04256106587712805,
+      "grad_norm": 0.1283423900604248,
+      "learning_rate": 1.802630258302583e-05,
+      "loss": 0.0154,
+      "mean_token_accuracy": 0.9934558510780335,
+      "num_tokens": 4301143.0,
+      "step": 115
+    },
+    {
+      "epoch": 0.04441154700222058,
+      "grad_norm": 0.5970118641853333,
+      "learning_rate": 1.8816929889298894e-05,
+      "loss": 0.0169,
+      "mean_token_accuracy": 0.9931488752365112,
+      "num_tokens": 4485870.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.0462620281273131,
+      "grad_norm": 0.1334739327430725,
+      "learning_rate": 1.9607557195571956e-05,
+      "loss": 0.0158,
+      "mean_token_accuracy": 0.9934451222419739,
+      "num_tokens": 4671825.0,
+      "step": 125
+    },
+    {
+      "epoch": 0.04811250925240563,
+      "grad_norm": 0.1656082421541214,
+      "learning_rate": 2.039818450184502e-05,
+      "loss": 0.0146,
+      "mean_token_accuracy": 0.9936710596084595,
+      "num_tokens": 4854954.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.04996299037749815,
+      "grad_norm": 0.10422646999359131,
+      "learning_rate": 2.1188811808118085e-05,
+      "loss": 0.0147,
+      "mean_token_accuracy": 0.9935644507408142,
+      "num_tokens": 5042206.0,
+      "step": 135
+    },
+    {
+      "epoch": 0.05181347150259067,
+      "grad_norm": 0.1137189045548439,
+      "learning_rate": 2.1979439114391147e-05,
+      "loss": 0.0147,
+      "mean_token_accuracy": 0.9937207102775574,
+      "num_tokens": 5231792.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.0536639526276832,
+      "grad_norm": 0.1359642595052719,
+      "learning_rate": 2.277006642066421e-05,
+      "loss": 0.0142,
+      "mean_token_accuracy": 0.993699848651886,
+      "num_tokens": 5412297.0,
+      "step": 145
+    },
+    {
+      "epoch": 0.05551443375277572,
+      "grad_norm": 0.17001201212406158,
+      "learning_rate": 2.356069372693727e-05,
+      "loss": 0.0138,
+      "mean_token_accuracy": 0.993944239616394,
+      "num_tokens": 5599717.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.057364914877868246,
+      "grad_norm": 0.11590602993965149,
+      "learning_rate": 2.435132103321033e-05,
+      "loss": 0.0136,
+      "mean_token_accuracy": 0.9938363671302796,
+      "num_tokens": 5786537.0,
+      "step": 155
+    },
+    {
+      "epoch": 0.05921539600296077,
+      "grad_norm": 0.11727731674909592,
+      "learning_rate": 2.5141948339483397e-05,
+      "loss": 0.0143,
+      "mean_token_accuracy": 0.993740177154541,
+      "num_tokens": 5969585.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.061065877128053295,
+      "grad_norm": 0.08245962113142014,
+      "learning_rate": 2.593257564575646e-05,
+      "loss": 0.0141,
+      "mean_token_accuracy": 0.9940990090370179,
+      "num_tokens": 6156724.0,
+      "step": 165
+    },
+    {
+      "epoch": 0.06291635825314582,
+      "grad_norm": 0.07970687001943588,
+      "learning_rate": 2.6723202952029522e-05,
+      "loss": 0.013,
+      "mean_token_accuracy": 0.9941760420799255,
+      "num_tokens": 6345575.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.06476683937823834,
+      "grad_norm": 0.20664170384407043,
+      "learning_rate": 2.751383025830258e-05,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9942710280418396,
+      "num_tokens": 6531385.0,
+      "step": 175
+    },
+    {
+      "epoch": 0.06661732050333087,
+      "grad_norm": 0.2238393872976303,
+      "learning_rate": 2.830445756457565e-05,
+      "loss": 0.0143,
+      "mean_token_accuracy": 0.9939396977424622,
+      "num_tokens": 6715295.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.06846780162842339,
+      "grad_norm": 0.16475221514701843,
+      "learning_rate": 2.909508487084871e-05,
+      "loss": 0.0139,
+      "mean_token_accuracy": 0.9937853693962098,
+      "num_tokens": 6901640.0,
+      "step": 185
+    },
+    {
+      "epoch": 0.07031828275351591,
+      "grad_norm": 0.11899226903915405,
+      "learning_rate": 2.9885712177121772e-05,
+      "loss": 0.0139,
+      "mean_token_accuracy": 0.9938597559928894,
+      "num_tokens": 7088485.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.07216876387860843,
+      "grad_norm": 0.05497577413916588,
+      "learning_rate": 3.0676339483394835e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9940391182899475,
+      "num_tokens": 7275336.0,
+      "step": 195
+    },
+    {
+      "epoch": 0.07401924500370097,
+      "grad_norm": 0.0667397677898407,
+      "learning_rate": 3.14669667896679e-05,
+      "loss": 0.0136,
+      "mean_token_accuracy": 0.994173550605774,
+      "num_tokens": 7462575.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.07586972612879349,
+      "grad_norm": 0.13350225985050201,
+      "learning_rate": 3.225759409594096e-05,
+      "loss": 0.0142,
+      "mean_token_accuracy": 0.9938544273376465,
+      "num_tokens": 7650192.0,
+      "step": 205
+    },
+    {
+      "epoch": 0.07772020725388601,
+      "grad_norm": 0.12862999737262726,
+      "learning_rate": 3.3048221402214026e-05,
+      "loss": 0.0129,
+      "mean_token_accuracy": 0.9943220376968384,
+      "num_tokens": 7841316.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.07957068837897853,
+      "grad_norm": 0.08196930587291718,
+      "learning_rate": 3.3838848708487085e-05,
+      "loss": 0.0135,
+      "mean_token_accuracy": 0.9939719796180725,
+      "num_tokens": 8022577.0,
+      "step": 215
+    },
+    {
+      "epoch": 0.08142116950407106,
+      "grad_norm": 0.11810460686683655,
+      "learning_rate": 3.462947601476015e-05,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.9945261120796204,
+      "num_tokens": 8211139.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.08327165062916358,
+      "grad_norm": 0.06291542947292328,
+      "learning_rate": 3.542010332103321e-05,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9944621562957764,
+      "num_tokens": 8397245.0,
+      "step": 225
+    },
+    {
+      "epoch": 0.0851221317542561,
+      "grad_norm": 0.06278537958860397,
+      "learning_rate": 3.6210730627306276e-05,
+      "loss": 0.014,
+      "mean_token_accuracy": 0.9939502358436585,
+      "num_tokens": 8580660.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.08697261287934863,
+      "grad_norm": 0.051632992923259735,
+      "learning_rate": 3.7001357933579335e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9946012616157531,
+      "num_tokens": 8768383.0,
+      "step": 235
+    },
+    {
+      "epoch": 0.08882309400444116,
+      "grad_norm": 0.06995566934347153,
+      "learning_rate": 3.77919852398524e-05,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9943013906478881,
+      "num_tokens": 8953368.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.09067357512953368,
+      "grad_norm": 0.1363731175661087,
+      "learning_rate": 3.858261254612547e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9944965839385986,
+      "num_tokens": 9138830.0,
+      "step": 245
+    },
+    {
+      "epoch": 0.0925240562546262,
+      "grad_norm": 0.05171045660972595,
+      "learning_rate": 3.9373239852398526e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9946675181388855,
+      "num_tokens": 9328907.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.09437453737971872,
+      "grad_norm": 0.05888763815164566,
+      "learning_rate": 4.0163867158671585e-05,
+      "loss": 0.0124,
+      "mean_token_accuracy": 0.9943785905838013,
+      "num_tokens": 9514714.0,
+      "step": 255
+    },
+    {
+      "epoch": 0.09622501850481126,
+      "grad_norm": 0.0596054270863533,
+      "learning_rate": 4.095449446494465e-05,
+      "loss": 0.0134,
+      "mean_token_accuracy": 0.9940859913825989,
+      "num_tokens": 9698135.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.09807549962990378,
+      "grad_norm": 0.08483681082725525,
+      "learning_rate": 4.174512177121772e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9941369533538819,
+      "num_tokens": 9882979.0,
+      "step": 265
+    },
+    {
+      "epoch": 0.0999259807549963,
+      "grad_norm": 0.07195551693439484,
+      "learning_rate": 4.2535749077490776e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.994370698928833,
+      "num_tokens": 10069134.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.10177646188008882,
+      "grad_norm": 0.040128063410520554,
+      "learning_rate": 4.285197291230987e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9942069530487061,
+      "num_tokens": 10257393.0,
+      "step": 275
+    },
+    {
+      "epoch": 0.10362694300518134,
+      "grad_norm": 0.03847283869981766,
+      "learning_rate": 4.285180737675649e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9942315220832825,
+      "num_tokens": 10446564.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.10547742413027388,
+      "grad_norm": 0.052056606858968735,
+      "learning_rate": 4.285149135591477e-05,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9941303849220275,
+      "num_tokens": 10635225.0,
+      "step": 285
+    },
+    {
+      "epoch": 0.1073279052553664,
+      "grad_norm": 0.11386790126562119,
+      "learning_rate": 4.285102485274419e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9943561673164367,
+      "num_tokens": 10824251.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.10917838638045892,
+      "grad_norm": 0.08637174218893051,
+      "learning_rate": 4.285040787161343e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9942921042442322,
+      "num_tokens": 11007473.0,
+      "step": 295
+    },
+    {
+      "epoch": 0.11102886750555144,
+      "grad_norm": 0.08053375780582428,
+      "learning_rate": 4.284964041830038e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9943666338920594,
+      "num_tokens": 11195505.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.11287934863064397,
+      "grad_norm": 0.053617142140865326,
+      "learning_rate": 4.284872249999205e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9943767070770264,
+      "num_tokens": 11383359.0,
+      "step": 305
+    },
+    {
+      "epoch": 0.11472982975573649,
+      "grad_norm": 0.236342191696167,
+      "learning_rate": 4.284765412528454e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9943820238113403,
+      "num_tokens": 11570508.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.11658031088082901,
+      "grad_norm": 0.0628127008676529,
+      "learning_rate": 4.2846435304182934e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9945466637611389,
+      "num_tokens": 11760407.0,
+      "step": 315
+    },
+    {
+      "epoch": 0.11843079200592153,
+      "grad_norm": 0.07259124517440796,
+      "learning_rate": 4.284506604810118e-05,
+      "loss": 0.0117,
+      "mean_token_accuracy": 0.994793963432312,
+      "num_tokens": 11946394.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.12028127313101407,
+      "grad_norm": 0.12081380188465118,
+      "learning_rate": 4.284354636986206e-05,
+      "loss": 0.0131,
+      "mean_token_accuracy": 0.9942123293876648,
+      "num_tokens": 12135829.0,
+      "step": 325
+    },
+    {
+      "epoch": 0.12213175425610659,
+      "grad_norm": 0.032616619020700455,
+      "learning_rate": 4.284187628369701e-05,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.9943543672561646,
+      "num_tokens": 12326292.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.12398223538119911,
+      "grad_norm": 0.0659327581524849,
+      "learning_rate": 4.284005580524598e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9946148633956909,
+      "num_tokens": 12512244.0,
+      "step": 335
+    },
+    {
+      "epoch": 0.12583271650629163,
+      "grad_norm": 0.0538957342505455,
+      "learning_rate": 4.283808495155733e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9943965435028076,
+      "num_tokens": 12698345.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.12768319763138417,
+      "grad_norm": 0.026756659150123596,
+      "learning_rate": 4.283596374108763e-05,
+      "loss": 0.0117,
+      "mean_token_accuracy": 0.9946674346923828,
+      "num_tokens": 12886124.0,
+      "step": 345
+    },
+    {
+      "epoch": 0.12953367875647667,
+      "grad_norm": 0.03937589004635811,
+      "learning_rate": 4.2833692193701535e-05,
+      "loss": 0.0124,
+      "mean_token_accuracy": 0.9941404342651368,
+      "num_tokens": 13072522.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.1313841598815692,
+      "grad_norm": 0.11673159897327423,
+      "learning_rate": 4.283127033067154e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.994466495513916,
+      "num_tokens": 13257550.0,
+      "step": 355
+    },
+    {
+      "epoch": 0.13323464100666174,
+      "grad_norm": 0.04208122193813324,
+      "learning_rate": 4.282869817467783e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9941461205482482,
+      "num_tokens": 13443219.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.13508512213175425,
+      "grad_norm": 0.0433068685233593,
+      "learning_rate": 4.282597574980804e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9947281122207642,
+      "num_tokens": 13630868.0,
+      "step": 365
+    },
+    {
+      "epoch": 0.13693560325684678,
+      "grad_norm": 0.11256636679172516,
+      "learning_rate": 4.2823103081557036e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.994697916507721,
+      "num_tokens": 13819698.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.13878608438193932,
+      "grad_norm": 0.05056292563676834,
+      "learning_rate": 4.282008019682668e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9944531917572021,
+      "num_tokens": 14006605.0,
+      "step": 375
+    },
+    {
+      "epoch": 0.14063656550703182,
+      "grad_norm": 0.04138778895139694,
+      "learning_rate": 4.2816907123925595e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9946407318115235,
+      "num_tokens": 14194590.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.14248704663212436,
+      "grad_norm": 0.04915003105998039,
+      "learning_rate": 4.2813583892568835e-05,
+      "loss": 0.0115,
+      "mean_token_accuracy": 0.9946125507354736,
+      "num_tokens": 14382780.0,
+      "step": 385
+    },
+    {
+      "epoch": 0.14433752775721687,
+      "grad_norm": 0.0317862406373024,
+      "learning_rate": 4.28101105338777e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9941527843475342,
+      "num_tokens": 14570415.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.1461880088823094,
+      "grad_norm": 0.04320110008120537,
+      "learning_rate": 4.280648708037937e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9945583343505859,
+      "num_tokens": 14757326.0,
+      "step": 395
+    },
+    {
+      "epoch": 0.14803849000740193,
+      "grad_norm": 0.0487859845161438,
+      "learning_rate": 4.280271356600664e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9947203397750854,
+      "num_tokens": 14939541.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.14988897113249444,
+      "grad_norm": 0.019251326099038124,
+      "learning_rate": 4.2798790026097594e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.994676387310028,
+      "num_tokens": 15126309.0,
+      "step": 405
+    },
+    {
+      "epoch": 0.15173945225758698,
+      "grad_norm": 0.06465357542037964,
+      "learning_rate": 4.2794716497395245e-05,
+      "loss": 0.0129,
+      "mean_token_accuracy": 0.9943367958068847,
+      "num_tokens": 15310070.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.15358993338267948,
+      "grad_norm": 0.05526921898126602,
+      "learning_rate": 4.2790493018047233e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9944483518600464,
+      "num_tokens": 15495949.0,
+      "step": 415
+    },
+    {
+      "epoch": 0.15544041450777202,
+      "grad_norm": 0.034128863364458084,
+      "learning_rate": 4.278611962760546e-05,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.9941602945327759,
+      "num_tokens": 15683861.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.15729089563286455,
+      "grad_norm": 0.040949802845716476,
+      "learning_rate": 4.2781596367025705e-05,
+      "loss": 0.0124,
+      "mean_token_accuracy": 0.9942546725273133,
+      "num_tokens": 15870718.0,
+      "step": 425
+    },
+    {
+      "epoch": 0.15914137675795706,
+      "grad_norm": 0.035726647824048996,
+      "learning_rate": 4.277692327866723e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9944644689559936,
+      "num_tokens": 16061189.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.1609918578830496,
+      "grad_norm": 0.04440707713365555,
+      "learning_rate": 4.27721004062924e-05,
+      "loss": 0.0131,
+      "mean_token_accuracy": 0.9941565513610839,
+      "num_tokens": 16244331.0,
+      "step": 435
+    },
+    {
+      "epoch": 0.16284233900814213,
+      "grad_norm": 0.04277065768837929,
+      "learning_rate": 4.27671277950663e-05,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9941763043403625,
+      "num_tokens": 16428979.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.16469282013323464,
+      "grad_norm": 0.05249185860157013,
+      "learning_rate": 4.276200549155626e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9945122838020325,
+      "num_tokens": 16616412.0,
+      "step": 445
+    },
+    {
+      "epoch": 0.16654330125832717,
+      "grad_norm": 0.05636998265981674,
+      "learning_rate": 4.2756733543731456e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9945295214653015,
+      "num_tokens": 16803515.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.16839378238341968,
+      "grad_norm": 0.022253001108765602,
+      "learning_rate": 4.275131200096242e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.994416344165802,
+      "num_tokens": 16989032.0,
+      "step": 455
+    },
+    {
+      "epoch": 0.1702442635085122,
+      "grad_norm": 0.032556477934122086,
+      "learning_rate": 4.2745740914020645e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9945114493370056,
+      "num_tokens": 17178481.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.17209474463360475,
+      "grad_norm": 0.057244472205638885,
+      "learning_rate": 4.2740020335078044e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9945224165916443,
+      "num_tokens": 17364736.0,
+      "step": 465
+    },
+    {
+      "epoch": 0.17394522575869725,
+      "grad_norm": 0.052654314786195755,
+      "learning_rate": 4.27341503177065e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9944727659225464,
+      "num_tokens": 17546812.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.1757957068837898,
+      "grad_norm": 0.06236141547560692,
+      "learning_rate": 4.2728130916877336e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9944144725799561,
+      "num_tokens": 17732195.0,
+      "step": 475
+    },
+    {
+      "epoch": 0.17764618800888232,
+      "grad_norm": 0.04595118761062622,
+      "learning_rate": 4.272196218896084e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9942219495773316,
+      "num_tokens": 17915325.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.17949666913397483,
+      "grad_norm": 0.023451240733265877,
+      "learning_rate": 4.2715644191725686e-05,
+      "loss": 0.0129,
+      "mean_token_accuracy": 0.9942497849464417,
+      "num_tokens": 18101917.0,
+      "step": 485
+    },
+    {
+      "epoch": 0.18134715025906736,
+      "grad_norm": 0.06435070186853409,
+      "learning_rate": 4.270917698433845e-05,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.9943255424499512,
+      "num_tokens": 18286995.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.18319763138415987,
+      "grad_norm": 0.020520444959402084,
+      "learning_rate": 4.270256062736299e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.994460117816925,
+      "num_tokens": 18473747.0,
+      "step": 495
+    },
+    {
+      "epoch": 0.1850481125092524,
+      "grad_norm": 0.060110948979854584,
+      "learning_rate": 4.269579518275996e-05,
+      "loss": 0.0117,
+      "mean_token_accuracy": 0.9946680665016174,
+      "num_tokens": 18661411.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.18689859363434494,
+      "grad_norm": 0.025209549814462662,
+      "learning_rate": 4.268888071388614e-05,
+      "loss": 0.0117,
+      "mean_token_accuracy": 0.9945947647094726,
+      "num_tokens": 18850199.0,
+      "step": 505
+    },
+    {
+      "epoch": 0.18874907475943745,
+      "grad_norm": 0.018915211781859398,
+      "learning_rate": 4.268181728549391e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9943953394889832,
+      "num_tokens": 19035469.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.19059955588452998,
+      "grad_norm": 0.02561143785715103,
+      "learning_rate": 4.267460496373061e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9942945837974548,
+      "num_tokens": 19217948.0,
+      "step": 515
+    },
+    {
+      "epoch": 0.19245003700962252,
+      "grad_norm": 0.0442853718996048,
+      "learning_rate": 4.266724381613796e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9944578647613526,
+      "num_tokens": 19404069.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.19430051813471502,
+      "grad_norm": 0.04662763699889183,
+      "learning_rate": 4.265973391165136e-05,
+      "loss": 0.0129,
+      "mean_token_accuracy": 0.9942412853240967,
+      "num_tokens": 19585254.0,
+      "step": 525
+    },
+    {
+      "epoch": 0.19615099925980756,
+      "grad_norm": 0.0530804768204689,
+      "learning_rate": 4.2652075320599303e-05,
+      "loss": 0.0117,
+      "mean_token_accuracy": 0.9946515679359436,
+      "num_tokens": 19775480.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.19800148038490006,
+      "grad_norm": 0.03273262456059456,
+      "learning_rate": 4.264426811470271e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9944777727127075,
+      "num_tokens": 19959932.0,
+      "step": 535
+    },
+    {
+      "epoch": 0.1998519615099926,
+      "grad_norm": 0.04370114952325821,
+      "learning_rate": 4.263631236707421e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9945811629295349,
+      "num_tokens": 20146524.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.20170244263508513,
+      "grad_norm": 0.023784466087818146,
+      "learning_rate": 4.262820815221752e-05,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.9943390250205993,
+      "num_tokens": 20332234.0,
+      "step": 545
+    },
+    {
+      "epoch": 0.20355292376017764,
+      "grad_norm": 0.8499415516853333,
+      "learning_rate": 4.261995554602671e-05,
+      "loss": 0.0353,
+      "mean_token_accuracy": 0.9902504801750183,
+      "num_tokens": 20514499.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.20540340488527017,
+      "grad_norm": 1.1610054969787598,
+      "learning_rate": 4.261155462578549e-05,
+      "loss": 0.0206,
+      "mean_token_accuracy": 0.9920717954635621,
+      "num_tokens": 20700594.0,
+      "step": 555
+    },
+    {
+      "epoch": 0.20725388601036268,
+      "grad_norm": 0.13676072657108307,
+      "learning_rate": 4.260300547016651e-05,
+      "loss": 0.0163,
+      "mean_token_accuracy": 0.9935830473899842,
+      "num_tokens": 20890167.0,
+      "step": 560
+    },
+    {
+      "epoch": 0.20910436713545522,
+      "grad_norm": 0.15704846382141113,
+      "learning_rate": 4.259430815923061e-05,
+      "loss": 0.0147,
+      "mean_token_accuracy": 0.9940526485443115,
+      "num_tokens": 21076844.0,
+      "step": 565
+    },
+    {
+      "epoch": 0.21095484826054775,
+      "grad_norm": 0.11786659061908722,
+      "learning_rate": 4.258546277442605e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9944088220596313,
+      "num_tokens": 21262899.0,
+      "step": 570
+    },
+    {
+      "epoch": 0.21280532938564026,
+      "grad_norm": 0.11840520799160004,
+      "learning_rate": 4.2576469398587796e-05,
+      "loss": 0.0144,
+      "mean_token_accuracy": 0.9937516927719117,
+      "num_tokens": 21448343.0,
+      "step": 575
+    },
+    {
+      "epoch": 0.2146558105107328,
+      "grad_norm": 0.08956237137317657,
+      "learning_rate": 4.2567328115936684e-05,
+      "loss": 0.0136,
+      "mean_token_accuracy": 0.9941320896148682,
+      "num_tokens": 21634019.0,
+      "step": 580
+    },
+    {
+      "epoch": 0.21650629163582533,
+      "grad_norm": 0.02889622002840042,
+      "learning_rate": 4.255803901207869e-05,
+      "loss": 0.0139,
+      "mean_token_accuracy": 0.9940349817276001,
+      "num_tokens": 21816536.0,
+      "step": 585
+    },
+    {
+      "epoch": 0.21835677276091783,
+      "grad_norm": 0.026375873014330864,
+      "learning_rate": 4.254860217400409e-05,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.9942944407463074,
+      "num_tokens": 22002114.0,
+      "step": 590
+    },
+    {
+      "epoch": 0.22020725388601037,
+      "grad_norm": 0.023582832887768745,
+      "learning_rate": 4.253901769008664e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9944105982780457,
+      "num_tokens": 22186004.0,
+      "step": 595
+    },
+    {
+      "epoch": 0.22205773501110287,
+      "grad_norm": 0.029931364580988884,
+      "learning_rate": 4.25292856500828e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9946899533271789,
+      "num_tokens": 22369126.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.2239082161361954,
+      "grad_norm": 0.03375266492366791,
+      "learning_rate": 4.251940614513083e-05,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9943894147872925,
+      "num_tokens": 22557508.0,
+      "step": 605
+    },
+    {
+      "epoch": 0.22575869726128794,
+      "grad_norm": 0.037536900490522385,
+      "learning_rate": 4.2509379267749967e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9944296598434448,
+      "num_tokens": 22740164.0,
+      "step": 610
+    },
+    {
+      "epoch": 0.22760917838638045,
+      "grad_norm": 0.02084357477724552,
+      "learning_rate": 4.249920511183957e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.994500744342804,
+      "num_tokens": 22926105.0,
+      "step": 615
+    },
+    {
+      "epoch": 0.22945965951147299,
+      "grad_norm": 0.018535546958446503,
+      "learning_rate": 4.248888377267821e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9944629311561585,
+      "num_tokens": 23109821.0,
+      "step": 620
+    },
+    {
+      "epoch": 0.23131014063656552,
+      "grad_norm": 0.03311913460493088,
+      "learning_rate": 4.247841534692281e-05,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9942508697509765,
+      "num_tokens": 23298500.0,
+      "step": 625
+    },
+    {
+      "epoch": 0.23316062176165803,
+      "grad_norm": 0.03697752580046654,
+      "learning_rate": 4.246779993260773e-05,
+      "loss": 0.0125,
+      "mean_token_accuracy": 0.9944444537162781,
+      "num_tokens": 23484827.0,
+      "step": 630
+    },
+    {
+      "epoch": 0.23501110288675056,
+      "grad_norm": 0.04822184517979622,
+      "learning_rate": 4.24570376291438e-05,
+      "loss": 0.013,
+      "mean_token_accuracy": 0.9942883372306823,
+      "num_tokens": 23672624.0,
+      "step": 635
+    },
+    {
+      "epoch": 0.23686158401184307,
+      "grad_norm": 0.015962179750204086,
+      "learning_rate": 4.2446128537317495e-05,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9940625190734863,
+      "num_tokens": 23855305.0,
+      "step": 640
+    },
+    {
+      "epoch": 0.2387120651369356,
+      "grad_norm": 0.023596221581101418,
+      "learning_rate": 4.243507275928988e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9944760918617248,
+      "num_tokens": 24039098.0,
+      "step": 645
+    },
+    {
+      "epoch": 0.24056254626202814,
+      "grad_norm": 0.04977529123425484,
+      "learning_rate": 4.2423870398595724e-05,
+      "loss": 0.0131,
+      "mean_token_accuracy": 0.9939121842384339,
+      "num_tokens": 24224255.0,
+      "step": 650
+    },
+    {
+      "epoch": 0.24241302738712064,
+      "grad_norm": 0.02917158044874668,
+      "learning_rate": 4.24125215601425e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9942941784858703,
+      "num_tokens": 24408183.0,
+      "step": 655
+    },
+    {
+      "epoch": 0.24426350851221318,
+      "grad_norm": 0.023342030122876167,
+      "learning_rate": 4.240102635020941e-05,
+      "loss": 0.0115,
+      "mean_token_accuracy": 0.9947278022766113,
+      "num_tokens": 24596912.0,
+      "step": 660
+    },
+    {
+      "epoch": 0.24611398963730569,
+      "grad_norm": 0.019149361178278923,
+      "learning_rate": 4.238938487644642e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9943392276763916,
+      "num_tokens": 24783474.0,
+      "step": 665
+    },
+    {
+      "epoch": 0.24796447076239822,
+      "grad_norm": 0.0178193561732769,
+      "learning_rate": 4.237759724787315e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9944109082221985,
+      "num_tokens": 24969623.0,
+      "step": 670
+    },
+    {
+      "epoch": 0.24981495188749075,
+      "grad_norm": 0.05727710947394371,
+      "learning_rate": 4.2365663574878014e-05,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9943978309631347,
+      "num_tokens": 25158466.0,
+      "step": 675
+    },
+    {
+      "epoch": 0.25166543301258326,
+      "grad_norm": 0.030884942039847374,
+      "learning_rate": 4.235358396921704e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9944790363311767,
+      "num_tokens": 25346048.0,
+      "step": 680
+    },
+    {
+      "epoch": 0.25351591413767577,
+      "grad_norm": 0.026175323873758316,
+      "learning_rate": 4.234135854401289e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9945067286491394,
+      "num_tokens": 25532502.0,
+      "step": 685
+    },
+    {
+      "epoch": 0.25536639526276833,
+      "grad_norm": 0.023625630885362625,
+      "learning_rate": 4.232898741375382e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9944512009620666,
+      "num_tokens": 25717818.0,
+      "step": 690
+    },
+    {
+      "epoch": 0.25721687638786084,
+      "grad_norm": 0.023306002840399742,
+      "learning_rate": 4.2316470694292536e-05,
+      "loss": 0.0115,
+      "mean_token_accuracy": 0.9947312355041504,
+      "num_tokens": 25905758.0,
+      "step": 695
+    },
+    {
+      "epoch": 0.25906735751295334,
+      "grad_norm": 0.04023386910557747,
+      "learning_rate": 4.230380850284518e-05,
+      "loss": 0.0116,
+      "mean_token_accuracy": 0.9947368025779724,
+      "num_tokens": 26096154.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.2609178386380459,
+      "grad_norm": 0.054880719631910324,
+      "learning_rate": 4.229100095799022e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9944626927375794,
+      "num_tokens": 26282576.0,
+      "step": 705
+    },
+    {
+      "epoch": 0.2627683197631384,
+      "grad_norm": 0.05358368158340454,
+      "learning_rate": 4.227804817966728e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.994605278968811,
+      "num_tokens": 26466539.0,
+      "step": 710
+    },
+    {
+      "epoch": 0.2646188008882309,
+      "grad_norm": 0.049045782536268234,
+      "learning_rate": 4.226495028917609e-05,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9942863821983338,
+      "num_tokens": 26656251.0,
+      "step": 715
+    },
+    {
+      "epoch": 0.2664692820133235,
+      "grad_norm": 0.0411219596862793,
+      "learning_rate": 4.225170740917533e-05,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9943994283676147,
+      "num_tokens": 26838459.0,
+      "step": 720
+    },
+    {
+      "epoch": 0.268319763138416,
+      "grad_norm": 0.030834250152111053,
+      "learning_rate": 4.223831966368145e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9942387580871582,
+      "num_tokens": 27019549.0,
+      "step": 725
+    },
+    {
+      "epoch": 0.2701702442635085,
+      "grad_norm": 0.07497315853834152,
+      "learning_rate": 4.222478717806755e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9943762421607971,
+      "num_tokens": 27205313.0,
+      "step": 730
+    },
+    {
+      "epoch": 0.27202072538860106,
+      "grad_norm": 0.05946696922183037,
+      "learning_rate": 4.2211110079062164e-05,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9942971229553222,
+      "num_tokens": 27391816.0,
+      "step": 735
+    },
+    {
+      "epoch": 0.27387120651369357,
+      "grad_norm": 0.03350157290697098,
+      "learning_rate": 4.219728849474812e-05,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9944700241088867,
+      "num_tokens": 27582143.0,
+      "step": 740
+    },
+    {
+      "epoch": 0.2757216876387861,
+      "grad_norm": 0.034784168004989624,
+      "learning_rate": 4.218332255456131e-05,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.994387972354889,
+      "num_tokens": 27769962.0,
+      "step": 745
+    },
+    {
+      "epoch": 0.27757216876387864,
+      "grad_norm": 0.02214915119111538,
+      "learning_rate": 4.2169212389289474e-05,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.994516122341156,
+      "num_tokens": 27954489.0,
+      "step": 750
+    },
+    {
+      "epoch": 0.27942264988897114,
+      "grad_norm": 0.019403742626309395,
+      "learning_rate": 4.215495813107101e-05,
+      "loss": 0.0117,
+      "mean_token_accuracy": 0.9947906136512756,
+      "num_tokens": 28146292.0,
+      "step": 755
+    },
+    {
+      "epoch": 0.28127313101406365,
+      "grad_norm": 0.02483655884861946,
+      "learning_rate": 4.214055991339368e-05,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9943630814552307,
+      "num_tokens": 28330741.0,
+      "step": 760
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5404,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.634430263784899e+17,
+  "train_batch_size": 20,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:188c17dc9f2b68af9384dff66e29440c7c1a2b437c09900b469c6097b14a9fd8
+size 6545