Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-70B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-70B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-70B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fea92fc3a620feb33173d9763c6ed791a125fecfe3831ff949db421528c9c42b
+size 3313653480

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,2114 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 208,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4476254880428314,
+      "epoch": 0.009685230024213076,
+      "grad_norm": 1.3588989973068237,
+      "learning_rate": 0.0,
+      "loss": 2.2350821495056152,
+      "mean_token_accuracy": 0.5031501688063145,
+      "num_tokens": 96574.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.4007079899311066,
+      "epoch": 0.01937046004842615,
+      "grad_norm": 1.2849924564361572,
+      "learning_rate": 3.125e-06,
+      "loss": 2.14509654045105,
+      "mean_token_accuracy": 0.5205462127923965,
+      "num_tokens": 191520.0,
+      "step": 2
+    },
+    {
+      "entropy": 1.4708476215600967,
+      "epoch": 0.029055690072639227,
+      "grad_norm": 1.2314231395721436,
+      "learning_rate": 6.25e-06,
+      "loss": 2.158306121826172,
+      "mean_token_accuracy": 0.5159582570195198,
+      "num_tokens": 287631.0,
+      "step": 3
+    },
+    {
+      "entropy": 1.5651142746210098,
+      "epoch": 0.0387409200968523,
+      "grad_norm": 0.9785442352294922,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 2.0755105018615723,
+      "mean_token_accuracy": 0.5181002020835876,
+      "num_tokens": 384739.0,
+      "step": 4
+    },
+    {
+      "entropy": 1.579958975315094,
+      "epoch": 0.048426150121065374,
+      "grad_norm": 0.7276659607887268,
+      "learning_rate": 1.25e-05,
+      "loss": 1.930289626121521,
+      "mean_token_accuracy": 0.5408441498875618,
+      "num_tokens": 481674.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.8359138667583466,
+      "epoch": 0.05811138014527845,
+      "grad_norm": 0.5072433352470398,
+      "learning_rate": 1.5625e-05,
+      "loss": 1.9374589920043945,
+      "mean_token_accuracy": 0.5299954265356064,
+      "num_tokens": 576090.0,
+      "step": 6
+    },
+    {
+      "entropy": 1.8671298921108246,
+      "epoch": 0.06779661016949153,
+      "grad_norm": 0.44714638590812683,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 1.8663867712020874,
+      "mean_token_accuracy": 0.5436736792325974,
+      "num_tokens": 673951.0,
+      "step": 7
+    },
+    {
+      "entropy": 1.933446392416954,
+      "epoch": 0.0774818401937046,
+      "grad_norm": 0.5136867761611938,
+      "learning_rate": 2.1875e-05,
+      "loss": 1.7743638753890991,
+      "mean_token_accuracy": 0.5584963485598564,
+      "num_tokens": 771899.0,
+      "step": 8
+    },
+    {
+      "entropy": 1.9850385338068008,
+      "epoch": 0.08716707021791767,
+      "grad_norm": 0.5442814826965332,
+      "learning_rate": 2.5e-05,
+      "loss": 1.7758773565292358,
+      "mean_token_accuracy": 0.5573008134961128,
+      "num_tokens": 868206.0,
+      "step": 9
+    },
+    {
+      "entropy": 1.9424707889556885,
+      "epoch": 0.09685230024213075,
+      "grad_norm": 0.5084043741226196,
+      "learning_rate": 2.8125000000000003e-05,
+      "loss": 1.7394293546676636,
+      "mean_token_accuracy": 0.5608929991722107,
+      "num_tokens": 964219.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.8063177168369293,
+      "epoch": 0.10653753026634383,
+      "grad_norm": 0.3967217206954956,
+      "learning_rate": 3.125e-05,
+      "loss": 1.6807222366333008,
+      "mean_token_accuracy": 0.5679514929652214,
+      "num_tokens": 1062125.0,
+      "step": 11
+    },
+    {
+      "entropy": 1.638314574956894,
+      "epoch": 0.1162227602905569,
+      "grad_norm": 0.5107802152633667,
+      "learning_rate": 3.4375e-05,
+      "loss": 1.6123173236846924,
+      "mean_token_accuracy": 0.583977460861206,
+      "num_tokens": 1159319.0,
+      "step": 12
+    },
+    {
+      "entropy": 1.5576547533273697,
+      "epoch": 0.12590799031476999,
+      "grad_norm": 0.3253207802772522,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.6140292882919312,
+      "mean_token_accuracy": 0.5774664506316185,
+      "num_tokens": 1256881.0,
+      "step": 13
+    },
+    {
+      "entropy": 1.4602049440145493,
+      "epoch": 0.13559322033898305,
+      "grad_norm": 0.3317743241786957,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 1.5576837062835693,
+      "mean_token_accuracy": 0.586665965616703,
+      "num_tokens": 1351620.0,
+      "step": 14
+    },
+    {
+      "entropy": 1.4226532131433487,
+      "epoch": 0.14527845036319612,
+      "grad_norm": 0.2991933524608612,
+      "learning_rate": 4.375e-05,
+      "loss": 1.5325368642807007,
+      "mean_token_accuracy": 0.5934261903166771,
+      "num_tokens": 1449026.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.453083336353302,
+      "epoch": 0.1549636803874092,
+      "grad_norm": 0.26260727643966675,
+      "learning_rate": 4.6875e-05,
+      "loss": 1.5117604732513428,
+      "mean_token_accuracy": 0.5941561385989189,
+      "num_tokens": 1546355.0,
+      "step": 16
+    },
+    {
+      "entropy": 1.4769755005836487,
+      "epoch": 0.16464891041162227,
+      "grad_norm": 0.24356529116630554,
+      "learning_rate": 5e-05,
+      "loss": 1.5000947713851929,
+      "mean_token_accuracy": 0.5963850915431976,
+      "num_tokens": 1644308.0,
+      "step": 17
+    },
+    {
+      "entropy": 1.4608993381261826,
+      "epoch": 0.17433414043583534,
+      "grad_norm": 0.27541783452033997,
+      "learning_rate": 4.999859193643945e-05,
+      "loss": 1.4296932220458984,
+      "mean_token_accuracy": 0.6135347038507462,
+      "num_tokens": 1741625.0,
+      "step": 18
+    },
+    {
+      "entropy": 1.4715198874473572,
+      "epoch": 0.18401937046004843,
+      "grad_norm": 0.24254612624645233,
+      "learning_rate": 4.999436790436924e-05,
+      "loss": 1.4109866619110107,
+      "mean_token_accuracy": 0.6162872388958931,
+      "num_tokens": 1839077.0,
+      "step": 19
+    },
+    {
+      "entropy": 1.4909484386444092,
+      "epoch": 0.1937046004842615,
+      "grad_norm": 0.24249622225761414,
+      "learning_rate": 4.9987328379605816e-05,
+      "loss": 1.4181320667266846,
+      "mean_token_accuracy": 0.6092976555228233,
+      "num_tokens": 1936754.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.468568578362465,
+      "epoch": 0.2033898305084746,
+      "grad_norm": 0.21892383694648743,
+      "learning_rate": 4.9977474155117045e-05,
+      "loss": 1.4207193851470947,
+      "mean_token_accuracy": 0.6143926084041595,
+      "num_tokens": 2034049.0,
+      "step": 21
+    },
+    {
+      "entropy": 1.4144536703824997,
+      "epoch": 0.21307506053268765,
+      "grad_norm": 0.24802649021148682,
+      "learning_rate": 4.9964806340932865e-05,
+      "loss": 1.3823838233947754,
+      "mean_token_accuracy": 0.6196955218911171,
+      "num_tokens": 2131193.0,
+      "step": 22
+    },
+    {
+      "entropy": 1.4027946889400482,
+      "epoch": 0.22276029055690072,
+      "grad_norm": 0.2114405333995819,
+      "learning_rate": 4.9949326364020314e-05,
+      "loss": 1.3788299560546875,
+      "mean_token_accuracy": 0.6205098181962967,
+      "num_tokens": 2228554.0,
+      "step": 23
+    },
+    {
+      "entropy": 1.3630587011575699,
+      "epoch": 0.2324455205811138,
+      "grad_norm": 0.18567043542861938,
+      "learning_rate": 4.993103596812268e-05,
+      "loss": 1.3441742658615112,
+      "mean_token_accuracy": 0.6255069524049759,
+      "num_tokens": 2325761.0,
+      "step": 24
+    },
+    {
+      "entropy": 1.3504581600427628,
+      "epoch": 0.24213075060532688,
+      "grad_norm": 0.18140752613544464,
+      "learning_rate": 4.9909937213563165e-05,
+      "loss": 1.3367418050765991,
+      "mean_token_accuracy": 0.6294924318790436,
+      "num_tokens": 2420230.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.3556479513645172,
+      "epoch": 0.25181598062953997,
+      "grad_norm": 0.19952309131622314,
+      "learning_rate": 4.988603247701277e-05,
+      "loss": 1.329459547996521,
+      "mean_token_accuracy": 0.6290309652686119,
+      "num_tokens": 2515756.0,
+      "step": 26
+    },
+    {
+      "entropy": 1.3174784928560257,
+      "epoch": 0.26150121065375304,
+      "grad_norm": 0.2040238082408905,
+      "learning_rate": 4.985932445122257e-05,
+      "loss": 1.3146334886550903,
+      "mean_token_accuracy": 0.6350805461406708,
+      "num_tokens": 2611239.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.337596833705902,
+      "epoch": 0.2711864406779661,
+      "grad_norm": 0.1924348771572113,
+      "learning_rate": 4.982981614472039e-05,
+      "loss": 1.325057864189148,
+      "mean_token_accuracy": 0.6268098577857018,
+      "num_tokens": 2708463.0,
+      "step": 28
+    },
+    {
+      "entropy": 1.295740321278572,
+      "epoch": 0.28087167070217917,
+      "grad_norm": 0.18018221855163574,
+      "learning_rate": 4.979751088147192e-05,
+      "loss": 1.2943141460418701,
+      "mean_token_accuracy": 0.6371493935585022,
+      "num_tokens": 2806105.0,
+      "step": 29
+    },
+    {
+      "entropy": 1.2824859917163849,
+      "epoch": 0.29055690072639223,
+      "grad_norm": 0.1904715597629547,
+      "learning_rate": 4.97624123005063e-05,
+      "loss": 1.282959222793579,
+      "mean_token_accuracy": 0.6380399465560913,
+      "num_tokens": 2903815.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.274334877729416,
+      "epoch": 0.30024213075060535,
+      "grad_norm": 0.17454974353313446,
+      "learning_rate": 4.972452435550614e-05,
+      "loss": 1.2828162908554077,
+      "mean_token_accuracy": 0.6363882124423981,
+      "num_tokens": 3000656.0,
+      "step": 31
+    },
+    {
+      "entropy": 1.298419788479805,
+      "epoch": 0.3099273607748184,
+      "grad_norm": 0.16989658772945404,
+      "learning_rate": 4.968385131436222e-05,
+      "loss": 1.283800482749939,
+      "mean_token_accuracy": 0.6383039951324463,
+      "num_tokens": 3098537.0,
+      "step": 32
+    },
+    {
+      "entropy": 1.3138496726751328,
+      "epoch": 0.3196125907990315,
+      "grad_norm": 0.1963842809200287,
+      "learning_rate": 4.9640397758692715e-05,
+      "loss": 1.287667989730835,
+      "mean_token_accuracy": 0.6366405189037323,
+      "num_tokens": 3195923.0,
+      "step": 33
+    },
+    {
+      "entropy": 1.316459745168686,
+      "epoch": 0.32929782082324455,
+      "grad_norm": 0.17209197580814362,
+      "learning_rate": 4.9594168583327094e-05,
+      "loss": 1.2872288227081299,
+      "mean_token_accuracy": 0.6377927586436272,
+      "num_tokens": 3293424.0,
+      "step": 34
+    },
+    {
+      "entropy": 1.2688211798667908,
+      "epoch": 0.3389830508474576,
+      "grad_norm": 0.1710345298051834,
+      "learning_rate": 4.954516899575473e-05,
+      "loss": 1.2326550483703613,
+      "mean_token_accuracy": 0.6454800367355347,
+      "num_tokens": 3390314.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.275676041841507,
+      "epoch": 0.3486682808716707,
+      "grad_norm": 0.16850367188453674,
+      "learning_rate": 4.949340451553833e-05,
+      "loss": 1.2480475902557373,
+      "mean_token_accuracy": 0.644576296210289,
+      "num_tokens": 3485697.0,
+      "step": 36
+    },
+    {
+      "entropy": 1.2488422840833664,
+      "epoch": 0.3583535108958838,
+      "grad_norm": 0.17575675249099731,
+      "learning_rate": 4.943888097369216e-05,
+      "loss": 1.2449251413345337,
+      "mean_token_accuracy": 0.6482485681772232,
+      "num_tokens": 3583116.0,
+      "step": 37
+    },
+    {
+      "entropy": 1.20504991710186,
+      "epoch": 0.36803874092009686,
+      "grad_norm": 0.16634824872016907,
+      "learning_rate": 4.938160451202521e-05,
+      "loss": 1.210385799407959,
+      "mean_token_accuracy": 0.6547724008560181,
+      "num_tokens": 3678517.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.236429587006569,
+      "epoch": 0.37772397094430993,
+      "grad_norm": 0.15360407531261444,
+      "learning_rate": 4.9321581582449365e-05,
+      "loss": 1.2314069271087646,
+      "mean_token_accuracy": 0.6504408493638039,
+      "num_tokens": 3775690.0,
+      "step": 39
+    },
+    {
+      "entropy": 1.2558289468288422,
+      "epoch": 0.387409200968523,
+      "grad_norm": 0.1626509577035904,
+      "learning_rate": 4.9258818946252624e-05,
+      "loss": 1.253812551498413,
+      "mean_token_accuracy": 0.6433829963207245,
+      "num_tokens": 3873239.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2188849300146103,
+      "epoch": 0.39709443099273606,
+      "grad_norm": 0.16926692426204681,
+      "learning_rate": 4.9193323673337476e-05,
+      "loss": 1.2143971920013428,
+      "mean_token_accuracy": 0.6519596204161644,
+      "num_tokens": 3970507.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.2296934127807617,
+      "epoch": 0.4067796610169492,
+      "grad_norm": 0.17094968259334564,
+      "learning_rate": 4.912510314142448e-05,
+      "loss": 1.2195372581481934,
+      "mean_token_accuracy": 0.6513471901416779,
+      "num_tokens": 4067145.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.2168416231870651,
+      "epoch": 0.41646489104116224,
+      "grad_norm": 0.15936042368412018,
+      "learning_rate": 4.9054165035221236e-05,
+      "loss": 1.2123249769210815,
+      "mean_token_accuracy": 0.6524514406919479,
+      "num_tokens": 4165113.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.1896271854639053,
+      "epoch": 0.4261501210653753,
+      "grad_norm": 0.15422837436199188,
+      "learning_rate": 4.898051734555676e-05,
+      "loss": 1.1957529783248901,
+      "mean_token_accuracy": 0.6586505621671677,
+      "num_tokens": 4262663.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.2026645243167877,
+      "epoch": 0.4358353510895884,
+      "grad_norm": 0.16097040474414825,
+      "learning_rate": 4.890416836848127e-05,
+      "loss": 1.1992558240890503,
+      "mean_token_accuracy": 0.6544606015086174,
+      "num_tokens": 4359900.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.1843629777431488,
+      "epoch": 0.44552058111380144,
+      "grad_norm": 0.1597595363855362,
+      "learning_rate": 4.88251267043318e-05,
+      "loss": 1.1683578491210938,
+      "mean_token_accuracy": 0.6638541519641876,
+      "num_tokens": 4456985.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.1679482907056808,
+      "epoch": 0.4552058111380145,
+      "grad_norm": 0.15203599631786346,
+      "learning_rate": 4.87434012567633e-05,
+      "loss": 1.1561784744262695,
+      "mean_token_accuracy": 0.6650881245732307,
+      "num_tokens": 4554634.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.2105818390846252,
+      "epoch": 0.4648910411622276,
+      "grad_norm": 0.15597011148929596,
+      "learning_rate": 4.865900123174574e-05,
+      "loss": 1.1950294971466064,
+      "mean_token_accuracy": 0.6544563546776772,
+      "num_tokens": 4652249.0,
+      "step": 48
+    },
+    {
+      "entropy": 1.2046389728784561,
+      "epoch": 0.4745762711864407,
+      "grad_norm": 0.15933600068092346,
+      "learning_rate": 4.857193613652711e-05,
+      "loss": 1.183128833770752,
+      "mean_token_accuracy": 0.6564416959881783,
+      "num_tokens": 4748832.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.2113090008497238,
+      "epoch": 0.48426150121065376,
+      "grad_norm": 0.15904678404331207,
+      "learning_rate": 4.848221577856243e-05,
+      "loss": 1.1915383338928223,
+      "mean_token_accuracy": 0.6567506939172745,
+      "num_tokens": 4845986.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1754019558429718,
+      "epoch": 0.4939467312348668,
+      "grad_norm": 0.15548165142536163,
+      "learning_rate": 4.838985026440905e-05,
+      "loss": 1.1645618677139282,
+      "mean_token_accuracy": 0.6654537692666054,
+      "num_tokens": 4942777.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.1717564463615417,
+      "epoch": 0.5036319612590799,
+      "grad_norm": 0.16495537757873535,
+      "learning_rate": 4.829484999858815e-05,
+      "loss": 1.1674176454544067,
+      "mean_token_accuracy": 0.659358873963356,
+      "num_tokens": 5040326.0,
+      "step": 52
+    },
+    {
+      "entropy": 1.1637324541807175,
+      "epoch": 0.513317191283293,
+      "grad_norm": 0.16075873374938965,
+      "learning_rate": 4.819722568241274e-05,
+      "loss": 1.1621849536895752,
+      "mean_token_accuracy": 0.663130484521389,
+      "num_tokens": 5137838.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.1703068912029266,
+      "epoch": 0.5230024213075061,
+      "grad_norm": 0.16048873960971832,
+      "learning_rate": 4.8096988312782174e-05,
+      "loss": 1.1667448282241821,
+      "mean_token_accuracy": 0.6606730446219444,
+      "num_tokens": 5234578.0,
+      "step": 54
+    },
+    {
+      "entropy": 1.1869463622570038,
+      "epoch": 0.5326876513317191,
+      "grad_norm": 0.1574316769838333,
+      "learning_rate": 4.799414918094347e-05,
+      "loss": 1.1867117881774902,
+      "mean_token_accuracy": 0.6562071144580841,
+      "num_tokens": 5332439.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.1833796501159668,
+      "epoch": 0.5423728813559322,
+      "grad_norm": 0.1627049446105957,
+      "learning_rate": 4.788871987121937e-05,
+      "loss": 1.1771697998046875,
+      "mean_token_accuracy": 0.6603464111685753,
+      "num_tokens": 5430102.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.1720363795757294,
+      "epoch": 0.5520581113801453,
+      "grad_norm": 0.30378302931785583,
+      "learning_rate": 4.77807122597034e-05,
+      "loss": 1.1528563499450684,
+      "mean_token_accuracy": 0.6617102548480034,
+      "num_tokens": 5528264.0,
+      "step": 57
+    },
+    {
+      "entropy": 1.1582888960838318,
+      "epoch": 0.5617433414043583,
+      "grad_norm": 0.1613432615995407,
+      "learning_rate": 4.767013851292212e-05,
+      "loss": 1.1482983827590942,
+      "mean_token_accuracy": 0.6675111055374146,
+      "num_tokens": 5625491.0,
+      "step": 58
+    },
+    {
+      "entropy": 1.1704413145780563,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.15665417909622192,
+      "learning_rate": 4.755701108646463e-05,
+      "loss": 1.1476081609725952,
+      "mean_token_accuracy": 0.6660670340061188,
+      "num_tokens": 5723563.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.1684788465499878,
+      "epoch": 0.5811138014527845,
+      "grad_norm": 0.1587826907634735,
+      "learning_rate": 4.744134272357948e-05,
+      "loss": 1.15497624874115,
+      "mean_token_accuracy": 0.662791483104229,
+      "num_tokens": 5821106.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1705324202775955,
+      "epoch": 0.5907990314769975,
+      "grad_norm": 0.1650344729423523,
+      "learning_rate": 4.732314645373921e-05,
+      "loss": 1.168868899345398,
+      "mean_token_accuracy": 0.659059427678585,
+      "num_tokens": 5918772.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.1340993344783783,
+      "epoch": 0.6004842615012107,
+      "grad_norm": 0.15585103631019592,
+      "learning_rate": 4.7202435591172676e-05,
+      "loss": 1.1181915998458862,
+      "mean_token_accuracy": 0.6716931834816933,
+      "num_tokens": 6016758.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.1135689467191696,
+      "epoch": 0.6101694915254238,
+      "grad_norm": 0.15927211940288544,
+      "learning_rate": 4.7079223733365237e-05,
+      "loss": 1.1100869178771973,
+      "mean_token_accuracy": 0.6781439930200577,
+      "num_tokens": 6113210.0,
+      "step": 63
+    },
+    {
+      "entropy": 1.1656545102596283,
+      "epoch": 0.6198547215496368,
+      "grad_norm": 0.1585935652256012,
+      "learning_rate": 4.6953524759527054e-05,
+      "loss": 1.1522451639175415,
+      "mean_token_accuracy": 0.664880745112896,
+      "num_tokens": 6210272.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.1241111010313034,
+      "epoch": 0.6295399515738499,
+      "grad_norm": 0.15607106685638428,
+      "learning_rate": 4.6825352829029705e-05,
+      "loss": 1.1200485229492188,
+      "mean_token_accuracy": 0.671033151447773,
+      "num_tokens": 6307273.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.135048821568489,
+      "epoch": 0.639225181598063,
+      "grad_norm": 0.1598539501428604,
+      "learning_rate": 4.6694722379811185e-05,
+      "loss": 1.1302683353424072,
+      "mean_token_accuracy": 0.6700672060251236,
+      "num_tokens": 6404865.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.1544559001922607,
+      "epoch": 0.648910411622276,
+      "grad_norm": 0.16644105315208435,
+      "learning_rate": 4.656164812674951e-05,
+      "loss": 1.1450002193450928,
+      "mean_token_accuracy": 0.6640813648700714,
+      "num_tokens": 6501650.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.132233053445816,
+      "epoch": 0.6585956416464891,
+      "grad_norm": 0.16045193374156952,
+      "learning_rate": 4.642614506000523e-05,
+      "loss": 1.121936559677124,
+      "mean_token_accuracy": 0.6716863736510277,
+      "num_tokens": 6599028.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.1332774758338928,
+      "epoch": 0.6682808716707022,
+      "grad_norm": 0.16355212032794952,
+      "learning_rate": 4.628822844333278e-05,
+      "loss": 1.1154568195343018,
+      "mean_token_accuracy": 0.6706111207604408,
+      "num_tokens": 6695913.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.117134228348732,
+      "epoch": 0.6779661016949152,
+      "grad_norm": 0.1609818935394287,
+      "learning_rate": 4.614791381236115e-05,
+      "loss": 1.108933925628662,
+      "mean_token_accuracy": 0.6739098057150841,
+      "num_tokens": 6793755.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.123718187212944,
+      "epoch": 0.6876513317191283,
+      "grad_norm": 0.16711370646953583,
+      "learning_rate": 4.6005216972843864e-05,
+      "loss": 1.1205769777297974,
+      "mean_token_accuracy": 0.6695492789149284,
+      "num_tokens": 6889869.0,
+      "step": 71
+    },
+    {
+      "entropy": 1.1330177187919617,
+      "epoch": 0.6973365617433414,
+      "grad_norm": 0.15766654908657074,
+      "learning_rate": 4.5860153998878494e-05,
+      "loss": 1.136279582977295,
+      "mean_token_accuracy": 0.6664912849664688,
+      "num_tokens": 6987242.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.1289773285388947,
+      "epoch": 0.7070217917675545,
+      "grad_norm": 0.167424738407135,
+      "learning_rate": 4.571274123109606e-05,
+      "loss": 1.1257307529449463,
+      "mean_token_accuracy": 0.6683443114161491,
+      "num_tokens": 7083683.0,
+      "step": 73
+    },
+    {
+      "entropy": 1.1089464724063873,
+      "epoch": 0.7167070217917676,
+      "grad_norm": 0.29150310158729553,
+      "learning_rate": 4.5562995274820284e-05,
+      "loss": 1.0986303091049194,
+      "mean_token_accuracy": 0.6785000637173653,
+      "num_tokens": 7180007.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.1168130189180374,
+      "epoch": 0.7263922518159807,
+      "grad_norm": 0.1623908281326294,
+      "learning_rate": 4.541093299819714e-05,
+      "loss": 1.1109479665756226,
+      "mean_token_accuracy": 0.6751653999090195,
+      "num_tokens": 7276880.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.1238265335559845,
+      "epoch": 0.7360774818401937,
+      "grad_norm": 0.16212861239910126,
+      "learning_rate": 4.5256571530294664e-05,
+      "loss": 1.104738473892212,
+      "mean_token_accuracy": 0.6755917444825172,
+      "num_tokens": 7374051.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.1079749763011932,
+      "epoch": 0.7457627118644068,
+      "grad_norm": 0.15648601949214935,
+      "learning_rate": 4.5099928259173516e-05,
+      "loss": 1.1000884771347046,
+      "mean_token_accuracy": 0.6749309673905373,
+      "num_tokens": 7471906.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.12865149974823,
+      "epoch": 0.7554479418886199,
+      "grad_norm": 0.15941032767295837,
+      "learning_rate": 4.4941020829928245e-05,
+      "loss": 1.1193838119506836,
+      "mean_token_accuracy": 0.6708522811532021,
+      "num_tokens": 7569981.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.1111949235200882,
+      "epoch": 0.7651331719128329,
+      "grad_norm": 0.16523951292037964,
+      "learning_rate": 4.477986714269972e-05,
+      "loss": 1.1221985816955566,
+      "mean_token_accuracy": 0.6717454493045807,
+      "num_tokens": 7663566.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.100884273648262,
+      "epoch": 0.774818401937046,
+      "grad_norm": 0.16432514786720276,
+      "learning_rate": 4.4616485350658685e-05,
+      "loss": 1.104296088218689,
+      "mean_token_accuracy": 0.6767150536179543,
+      "num_tokens": 7761120.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1229810565710068,
+      "epoch": 0.784503631961259,
+      "grad_norm": 0.17101751267910004,
+      "learning_rate": 4.445089385796099e-05,
+      "loss": 1.1202152967453003,
+      "mean_token_accuracy": 0.6708082109689713,
+      "num_tokens": 7859307.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.134630709886551,
+      "epoch": 0.7941888619854721,
+      "grad_norm": 0.1734648197889328,
+      "learning_rate": 4.4283111317674374e-05,
+      "loss": 1.1201170682907104,
+      "mean_token_accuracy": 0.6716241240501404,
+      "num_tokens": 7956863.0,
+      "step": 82
+    },
+    {
+      "entropy": 1.1259192377328873,
+      "epoch": 0.8038740920096852,
+      "grad_norm": 0.16475871205329895,
+      "learning_rate": 4.4113156629677316e-05,
+      "loss": 1.1068096160888672,
+      "mean_token_accuracy": 0.674552284181118,
+      "num_tokens": 8053100.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.0983240008354187,
+      "epoch": 0.8135593220338984,
+      "grad_norm": 0.17292368412017822,
+      "learning_rate": 4.394104893853007e-05,
+      "loss": 1.0763158798217773,
+      "mean_token_accuracy": 0.6806622296571732,
+      "num_tokens": 8151004.0,
+      "step": 84
+    },
+    {
+      "entropy": 1.1288987547159195,
+      "epoch": 0.8232445520581114,
+      "grad_norm": 0.16203227639198303,
+      "learning_rate": 4.3766807631318106e-05,
+      "loss": 1.1195769309997559,
+      "mean_token_accuracy": 0.671344555914402,
+      "num_tokens": 8248668.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.1410782486200333,
+      "epoch": 0.8329297820823245,
+      "grad_norm": 0.16533830761909485,
+      "learning_rate": 4.359045233546827e-05,
+      "loss": 1.1256823539733887,
+      "mean_token_accuracy": 0.6698467060923576,
+      "num_tokens": 8345497.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.0999096632003784,
+      "epoch": 0.8426150121065376,
+      "grad_norm": 0.17075778543949127,
+      "learning_rate": 4.341200291653781e-05,
+      "loss": 1.0872827768325806,
+      "mean_token_accuracy": 0.6762653961777687,
+      "num_tokens": 8443405.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.0918122231960297,
+      "epoch": 0.8523002421307506,
+      "grad_norm": 0.1702580451965332,
+      "learning_rate": 4.323147947597666e-05,
+      "loss": 1.0806095600128174,
+      "mean_token_accuracy": 0.6800353080034256,
+      "num_tokens": 8540868.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.0941280871629715,
+      "epoch": 0.8619854721549637,
+      "grad_norm": 0.16791851818561554,
+      "learning_rate": 4.3048902348863116e-05,
+      "loss": 1.0981943607330322,
+      "mean_token_accuracy": 0.6758224442601204,
+      "num_tokens": 8638186.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.0951679199934006,
+      "epoch": 0.8716707021791767,
+      "grad_norm": 0.1697986125946045,
+      "learning_rate": 4.2864292101613136e-05,
+      "loss": 1.0987718105316162,
+      "mean_token_accuracy": 0.6747072339057922,
+      "num_tokens": 8735354.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1223643571138382,
+      "epoch": 0.8813559322033898,
+      "grad_norm": 0.2108999490737915,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 1.1219862699508667,
+      "mean_token_accuracy": 0.6695899069309235,
+      "num_tokens": 8832051.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.1099307239055634,
+      "epoch": 0.8910411622276029,
+      "grad_norm": 0.18003661930561066,
+      "learning_rate": 4.248905565513023e-05,
+      "loss": 1.0990370512008667,
+      "mean_token_accuracy": 0.6750800833106041,
+      "num_tokens": 8929471.0,
+      "step": 92
+    },
+    {
+      "entropy": 1.1097175031900406,
+      "epoch": 0.9007263922518159,
+      "grad_norm": 0.18473289906978607,
+      "learning_rate": 4.229847172443866e-05,
+      "loss": 1.0890154838562012,
+      "mean_token_accuracy": 0.6764732897281647,
+      "num_tokens": 9026545.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.1048423945903778,
+      "epoch": 0.910411622276029,
+      "grad_norm": 0.17208412289619446,
+      "learning_rate": 4.210593920593201e-05,
+      "loss": 1.1022264957427979,
+      "mean_token_accuracy": 0.6741294264793396,
+      "num_tokens": 9124261.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.0943758487701416,
+      "epoch": 0.9200968523002422,
+      "grad_norm": 0.18163490295410156,
+      "learning_rate": 4.191147978745218e-05,
+      "loss": 1.0901988744735718,
+      "mean_token_accuracy": 0.6788006126880646,
+      "num_tokens": 9220673.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.1178465783596039,
+      "epoch": 0.9297820823244553,
+      "grad_norm": 0.1731623411178589,
+      "learning_rate": 4.171511537389684e-05,
+      "loss": 1.1111767292022705,
+      "mean_token_accuracy": 0.6736986339092255,
+      "num_tokens": 9318321.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.0975633561611176,
+      "epoch": 0.9394673123486683,
+      "grad_norm": 0.18208175897598267,
+      "learning_rate": 4.151686808475204e-05,
+      "loss": 1.0915802717208862,
+      "mean_token_accuracy": 0.6765245422720909,
+      "num_tokens": 9415394.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.114044964313507,
+      "epoch": 0.9491525423728814,
+      "grad_norm": 0.17058847844600677,
+      "learning_rate": 4.131676025160047e-05,
+      "loss": 1.1074469089508057,
+      "mean_token_accuracy": 0.6720306798815727,
+      "num_tokens": 9512880.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.07672019302845,
+      "epoch": 0.9588377723970944,
+      "grad_norm": 0.18478929996490479,
+      "learning_rate": 4.111481441560598e-05,
+      "loss": 1.0674448013305664,
+      "mean_token_accuracy": 0.6838654279708862,
+      "num_tokens": 9606053.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.0935327857732773,
+      "epoch": 0.9685230024213075,
+      "grad_norm": 0.17510797083377838,
+      "learning_rate": 4.091105332497439e-05,
+      "loss": 1.0954351425170898,
+      "mean_token_accuracy": 0.6775561571121216,
+      "num_tokens": 9702569.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.0855822414159775,
+      "epoch": 0.9782082324455206,
+      "grad_norm": 0.18510136008262634,
+      "learning_rate": 4.070549993239106e-05,
+      "loss": 1.0844038724899292,
+      "mean_token_accuracy": 0.681103028357029,
+      "num_tokens": 9799976.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.1072624325752258,
+      "epoch": 0.9878934624697336,
+      "grad_norm": 0.2109905481338501,
+      "learning_rate": 4.049817739243532e-05,
+      "loss": 1.1046338081359863,
+      "mean_token_accuracy": 0.6760703399777412,
+      "num_tokens": 9895575.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.0761565268039703,
+      "epoch": 0.9975786924939467,
+      "grad_norm": 0.18559105694293976,
+      "learning_rate": 4.028910905897229e-05,
+      "loss": 1.0674762725830078,
+      "mean_token_accuracy": 0.6826195642352104,
+      "num_tokens": 9992394.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.1065305471420288,
+      "epoch": 1.0,
+      "grad_norm": 0.3376278877258301,
+      "learning_rate": 4.007831848252211e-05,
+      "loss": 1.09256911277771,
+      "mean_token_accuracy": 0.678756445646286,
+      "num_tokens": 10016415.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.1107248812913895,
+      "epoch": 1.0096852300242132,
+      "grad_norm": 0.1708710938692093,
+      "learning_rate": 3.986582940760717e-05,
+      "loss": 1.0717049837112427,
+      "mean_token_accuracy": 0.6800583377480507,
+      "num_tokens": 10113793.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.0776572525501251,
+      "epoch": 1.0193704600484261,
+      "grad_norm": 0.17748922109603882,
+      "learning_rate": 3.965166577007733e-05,
+      "loss": 1.0392060279846191,
+      "mean_token_accuracy": 0.6892436370253563,
+      "num_tokens": 10211074.0,
+      "step": 106
+    },
+    {
+      "entropy": 1.0571173429489136,
+      "epoch": 1.0290556900726393,
+      "grad_norm": 0.18791398406028748,
+      "learning_rate": 3.94358516944137e-05,
+      "loss": 1.038546085357666,
+      "mean_token_accuracy": 0.6879089176654816,
+      "num_tokens": 10307987.0,
+      "step": 107
+    },
+    {
+      "entropy": 1.0400206819176674,
+      "epoch": 1.0387409200968523,
+      "grad_norm": 0.16869165003299713,
+      "learning_rate": 3.9218411491011176e-05,
+      "loss": 1.0324063301086426,
+      "mean_token_accuracy": 0.6877342015504837,
+      "num_tokens": 10405148.0,
+      "step": 108
+    },
+    {
+      "entropy": 1.0476296544075012,
+      "epoch": 1.0484261501210654,
+      "grad_norm": 0.18576034903526306,
+      "learning_rate": 3.899936965343989e-05,
+      "loss": 1.0663259029388428,
+      "mean_token_accuracy": 0.6821715012192726,
+      "num_tokens": 10502344.0,
+      "step": 109
+    },
+    {
+      "entropy": 0.9960673823952675,
+      "epoch": 1.0581113801452784,
+      "grad_norm": 0.17832568287849426,
+      "learning_rate": 3.877875085568623e-05,
+      "loss": 1.0000097751617432,
+      "mean_token_accuracy": 0.6989128738641739,
+      "num_tokens": 10599599.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.0352009981870651,
+      "epoch": 1.0677966101694916,
+      "grad_norm": 0.17328514158725739,
+      "learning_rate": 3.8556579949373384e-05,
+      "loss": 1.0506186485290527,
+      "mean_token_accuracy": 0.6840933337807655,
+      "num_tokens": 10694645.0,
+      "step": 111
+    },
+    {
+      "entropy": 1.0331084430217743,
+      "epoch": 1.0774818401937045,
+      "grad_norm": 0.18105825781822205,
+      "learning_rate": 3.833288196096194e-05,
+      "loss": 1.0261142253875732,
+      "mean_token_accuracy": 0.6913967728614807,
+      "num_tokens": 10790067.0,
+      "step": 112
+    },
+    {
+      "entropy": 1.0757844671607018,
+      "epoch": 1.0871670702179177,
+      "grad_norm": 0.1806410551071167,
+      "learning_rate": 3.8107682088930794e-05,
+      "loss": 1.0621310472488403,
+      "mean_token_accuracy": 0.6843468695878983,
+      "num_tokens": 10885655.0,
+      "step": 113
+    },
+    {
+      "entropy": 1.0726244300603867,
+      "epoch": 1.0968523002421307,
+      "grad_norm": 0.17602035403251648,
+      "learning_rate": 3.7881005700938635e-05,
+      "loss": 1.0586791038513184,
+      "mean_token_accuracy": 0.6825271546840668,
+      "num_tokens": 10983568.0,
+      "step": 114
+    },
+    {
+      "entropy": 1.0513464957475662,
+      "epoch": 1.1065375302663438,
+      "grad_norm": 0.1740947663784027,
+      "learning_rate": 3.7652878330966415e-05,
+      "loss": 1.035830020904541,
+      "mean_token_accuracy": 0.6891666278243065,
+      "num_tokens": 11081209.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.0637426376342773,
+      "epoch": 1.116222760290557,
+      "grad_norm": 0.1756405085325241,
+      "learning_rate": 3.742332567644107e-05,
+      "loss": 1.0529723167419434,
+      "mean_token_accuracy": 0.6864708811044693,
+      "num_tokens": 11178554.0,
+      "step": 116
+    },
+    {
+      "entropy": 1.032620556652546,
+      "epoch": 1.12590799031477,
+      "grad_norm": 0.1746027022600174,
+      "learning_rate": 3.719237359534087e-05,
+      "loss": 1.0131734609603882,
+      "mean_token_accuracy": 0.6930168569087982,
+      "num_tokens": 11275773.0,
+      "step": 117
+    },
+    {
+      "entropy": 1.0357091426849365,
+      "epoch": 1.1355932203389831,
+      "grad_norm": 0.17611640691757202,
+      "learning_rate": 3.6960048103282566e-05,
+      "loss": 1.0305650234222412,
+      "mean_token_accuracy": 0.6928873062133789,
+      "num_tokens": 11372312.0,
+      "step": 118
+    },
+    {
+      "entropy": 1.0247548967599869,
+      "epoch": 1.145278450363196,
+      "grad_norm": 0.16733205318450928,
+      "learning_rate": 3.672637537059093e-05,
+      "loss": 1.0267465114593506,
+      "mean_token_accuracy": 0.6941879317164421,
+      "num_tokens": 11469895.0,
+      "step": 119
+    },
+    {
+      "entropy": 1.0328287035226822,
+      "epoch": 1.1549636803874093,
+      "grad_norm": 0.17758148908615112,
+      "learning_rate": 3.6491381719350756e-05,
+      "loss": 1.038550853729248,
+      "mean_token_accuracy": 0.687534786760807,
+      "num_tokens": 11567819.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.0606635063886642,
+      "epoch": 1.1646489104116222,
+      "grad_norm": 0.18219131231307983,
+      "learning_rate": 3.6255093620441834e-05,
+      "loss": 1.0544192790985107,
+      "mean_token_accuracy": 0.6846267804503441,
+      "num_tokens": 11665799.0,
+      "step": 121
+    },
+    {
+      "entropy": 1.0355539470911026,
+      "epoch": 1.1743341404358354,
+      "grad_norm": 0.18732059001922607,
+      "learning_rate": 3.6017537690557115e-05,
+      "loss": 1.0224016904830933,
+      "mean_token_accuracy": 0.6915983185172081,
+      "num_tokens": 11762356.0,
+      "step": 122
+    },
+    {
+      "entropy": 1.085768237709999,
+      "epoch": 1.1840193704600483,
+      "grad_norm": 0.20525461435317993,
+      "learning_rate": 3.577874068920446e-05,
+      "loss": 1.057420253753662,
+      "mean_token_accuracy": 0.6817465797066689,
+      "num_tokens": 11857832.0,
+      "step": 123
+    },
+    {
+      "entropy": 1.024247646331787,
+      "epoch": 1.1937046004842615,
+      "grad_norm": 0.17486801743507385,
+      "learning_rate": 3.553872951569236e-05,
+      "loss": 1.007400393486023,
+      "mean_token_accuracy": 0.6967773362994194,
+      "num_tokens": 11953563.0,
+      "step": 124
+    },
+    {
+      "entropy": 1.0304122641682625,
+      "epoch": 1.2033898305084745,
+      "grad_norm": 0.17227725684642792,
+      "learning_rate": 3.529753120609982e-05,
+      "loss": 1.0166981220245361,
+      "mean_token_accuracy": 0.6900071352720261,
+      "num_tokens": 12051630.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.0403276681900024,
+      "epoch": 1.2130750605326877,
+      "grad_norm": 0.17845618724822998,
+      "learning_rate": 3.505517293023088e-05,
+      "loss": 1.0411816835403442,
+      "mean_token_accuracy": 0.6878266781568527,
+      "num_tokens": 12149525.0,
+      "step": 126
+    },
+    {
+      "entropy": 1.0372891202569008,
+      "epoch": 1.2227602905569008,
+      "grad_norm": 0.17896883189678192,
+      "learning_rate": 3.481168198855409e-05,
+      "loss": 1.0509088039398193,
+      "mean_token_accuracy": 0.6861589029431343,
+      "num_tokens": 12246944.0,
+      "step": 127
+    },
+    {
+      "entropy": 1.0567465275526047,
+      "epoch": 1.2324455205811138,
+      "grad_norm": 0.1735103726387024,
+      "learning_rate": 3.456708580912725e-05,
+      "loss": 1.048382043838501,
+      "mean_token_accuracy": 0.6845468729734421,
+      "num_tokens": 12343964.0,
+      "step": 128
+    },
+    {
+      "entropy": 1.044258013367653,
+      "epoch": 1.242130750605327,
+      "grad_norm": 0.179740771651268,
+      "learning_rate": 3.432141194450772e-05,
+      "loss": 1.0318509340286255,
+      "mean_token_accuracy": 0.6892603412270546,
+      "num_tokens": 12441245.0,
+      "step": 129
+    },
+    {
+      "entropy": 1.0394302904605865,
+      "epoch": 1.25181598062954,
+      "grad_norm": 0.18108829855918884,
+      "learning_rate": 3.407468806864883e-05,
+      "loss": 1.0396722555160522,
+      "mean_token_accuracy": 0.6859570071101189,
+      "num_tokens": 12538938.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.0346315652132034,
+      "epoch": 1.261501210653753,
+      "grad_norm": 0.16831137239933014,
+      "learning_rate": 3.382694197378252e-05,
+      "loss": 1.0301259756088257,
+      "mean_token_accuracy": 0.6876077353954315,
+      "num_tokens": 12636750.0,
+      "step": 131
+    },
+    {
+      "entropy": 1.0501351803541183,
+      "epoch": 1.271186440677966,
+      "grad_norm": 0.18215049803256989,
+      "learning_rate": 3.357820156728867e-05,
+      "loss": 1.0379700660705566,
+      "mean_token_accuracy": 0.6864259541034698,
+      "num_tokens": 12732510.0,
+      "step": 132
+    },
+    {
+      "entropy": 1.056677907705307,
+      "epoch": 1.2808716707021792,
+      "grad_norm": 0.17387507855892181,
+      "learning_rate": 3.332849486855144e-05,
+      "loss": 1.0349537134170532,
+      "mean_token_accuracy": 0.6882437765598297,
+      "num_tokens": 12828202.0,
+      "step": 133
+    },
+    {
+      "entropy": 1.0694306939840317,
+      "epoch": 1.2905569007263922,
+      "grad_norm": 0.18142169713974,
+      "learning_rate": 3.307785000580313e-05,
+      "loss": 1.0491408109664917,
+      "mean_token_accuracy": 0.6856276169419289,
+      "num_tokens": 12924452.0,
+      "step": 134
+    },
+    {
+      "entropy": 1.0445284843444824,
+      "epoch": 1.3002421307506054,
+      "grad_norm": 0.17338253557682037,
+      "learning_rate": 3.282629521295556e-05,
+      "loss": 1.0387568473815918,
+      "mean_token_accuracy": 0.6878345832228661,
+      "num_tokens": 13021226.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.03611271828413,
+      "epoch": 1.3099273607748185,
+      "grad_norm": 0.17574281990528107,
+      "learning_rate": 3.257385882641971e-05,
+      "loss": 1.032334566116333,
+      "mean_token_accuracy": 0.6887728422880173,
+      "num_tokens": 13117072.0,
+      "step": 136
+    },
+    {
+      "entropy": 1.0164377763867378,
+      "epoch": 1.3196125907990315,
+      "grad_norm": 0.17293879389762878,
+      "learning_rate": 3.232056928191376e-05,
+      "loss": 1.0038666725158691,
+      "mean_token_accuracy": 0.6958679780364037,
+      "num_tokens": 13214927.0,
+      "step": 137
+    },
+    {
+      "entropy": 1.02716813236475,
+      "epoch": 1.3292978208232444,
+      "grad_norm": 0.1768789142370224,
+      "learning_rate": 3.2066455111259945e-05,
+      "loss": 1.0230392217636108,
+      "mean_token_accuracy": 0.6910659521818161,
+      "num_tokens": 13312250.0,
+      "step": 138
+    },
+    {
+      "entropy": 1.0242123305797577,
+      "epoch": 1.3389830508474576,
+      "grad_norm": 0.18655547499656677,
+      "learning_rate": 3.1811544939170575e-05,
+      "loss": 1.027778148651123,
+      "mean_token_accuracy": 0.6896005198359489,
+      "num_tokens": 13409125.0,
+      "step": 139
+    },
+    {
+      "entropy": 1.0247902870178223,
+      "epoch": 1.3486682808716708,
+      "grad_norm": 0.1732546091079712,
+      "learning_rate": 3.155586748002362e-05,
+      "loss": 1.0247126817703247,
+      "mean_token_accuracy": 0.6925645992159843,
+      "num_tokens": 13505714.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.0391114205121994,
+      "epoch": 1.3583535108958837,
+      "grad_norm": 0.1730506271123886,
+      "learning_rate": 3.1299451534628135e-05,
+      "loss": 1.0413163900375366,
+      "mean_token_accuracy": 0.6873403564095497,
+      "num_tokens": 13603282.0,
+      "step": 141
+    },
+    {
+      "entropy": 1.0580826699733734,
+      "epoch": 1.368038740920097,
+      "grad_norm": 0.17985138297080994,
+      "learning_rate": 3.1042325986980064e-05,
+      "loss": 1.048020362854004,
+      "mean_token_accuracy": 0.6840562298893929,
+      "num_tokens": 13699120.0,
+      "step": 142
+    },
+    {
+      "entropy": 1.034823827445507,
+      "epoch": 1.3777239709443099,
+      "grad_norm": 0.1742866039276123,
+      "learning_rate": 3.0784519801008546e-05,
+      "loss": 1.019952416419983,
+      "mean_token_accuracy": 0.6919118836522102,
+      "num_tokens": 13796639.0,
+      "step": 143
+    },
+    {
+      "entropy": 1.034243457019329,
+      "epoch": 1.387409200968523,
+      "grad_norm": 0.18104857206344604,
+      "learning_rate": 3.0526062017313254e-05,
+      "loss": 1.0257322788238525,
+      "mean_token_accuracy": 0.693074606359005,
+      "num_tokens": 13893968.0,
+      "step": 144
+    },
+    {
+      "entropy": 1.0477502346038818,
+      "epoch": 1.397094430992736,
+      "grad_norm": 0.17592455446720123,
+      "learning_rate": 3.0266981749893157e-05,
+      "loss": 1.0342764854431152,
+      "mean_token_accuracy": 0.6866420730948448,
+      "num_tokens": 13990892.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.0479690209031105,
+      "epoch": 1.4067796610169492,
+      "grad_norm": 0.1820760816335678,
+      "learning_rate": 3.0007308182866985e-05,
+      "loss": 1.0341622829437256,
+      "mean_token_accuracy": 0.687219500541687,
+      "num_tokens": 14087344.0,
+      "step": 146
+    },
+    {
+      "entropy": 1.0348835960030556,
+      "epoch": 1.4164648910411621,
+      "grad_norm": 0.1742199808359146,
+      "learning_rate": 2.974707056718571e-05,
+      "loss": 1.0271401405334473,
+      "mean_token_accuracy": 0.6896429732441902,
+      "num_tokens": 14185593.0,
+      "step": 147
+    },
+    {
+      "entropy": 1.0333365574479103,
+      "epoch": 1.4261501210653753,
+      "grad_norm": 0.177699476480484,
+      "learning_rate": 2.9486298217337637e-05,
+      "loss": 1.0313661098480225,
+      "mean_token_accuracy": 0.6886806860566139,
+      "num_tokens": 14283694.0,
+      "step": 148
+    },
+    {
+      "entropy": 1.0181496515870094,
+      "epoch": 1.4358353510895885,
+      "grad_norm": 0.17134729027748108,
+      "learning_rate": 2.9225020508046232e-05,
+      "loss": 1.009653091430664,
+      "mean_token_accuracy": 0.6931144669651985,
+      "num_tokens": 14380604.0,
+      "step": 149
+    },
+    {
+      "entropy": 1.038355827331543,
+      "epoch": 1.4455205811138014,
+      "grad_norm": 0.1845579445362091,
+      "learning_rate": 2.8963266870961226e-05,
+      "loss": 1.0307377576828003,
+      "mean_token_accuracy": 0.6884006261825562,
+      "num_tokens": 14478185.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.0334223732352257,
+      "epoch": 1.4552058111380144,
+      "grad_norm": 0.24978552758693695,
+      "learning_rate": 2.8701066791343288e-05,
+      "loss": 1.0366319417953491,
+      "mean_token_accuracy": 0.6860118135809898,
+      "num_tokens": 14575277.0,
+      "step": 151
+    },
+    {
+      "entropy": 1.0341607704758644,
+      "epoch": 1.4648910411622276,
+      "grad_norm": 0.17872564494609833,
+      "learning_rate": 2.8438449804742628e-05,
+      "loss": 1.0180070400238037,
+      "mean_token_accuracy": 0.6917398646473885,
+      "num_tokens": 14672534.0,
+      "step": 152
+    },
+    {
+      "entropy": 1.0590864270925522,
+      "epoch": 1.4745762711864407,
+      "grad_norm": 0.17723426222801208,
+      "learning_rate": 2.8175445493671972e-05,
+      "loss": 1.051386833190918,
+      "mean_token_accuracy": 0.6834967508912086,
+      "num_tokens": 14770518.0,
+      "step": 153
+    },
+    {
+      "entropy": 1.0175726786255836,
+      "epoch": 1.4842615012106537,
+      "grad_norm": 0.17911297082901,
+      "learning_rate": 2.7912083484274266e-05,
+      "loss": 0.9993636012077332,
+      "mean_token_accuracy": 0.6968463584780693,
+      "num_tokens": 14867845.0,
+      "step": 154
+    },
+    {
+      "entropy": 1.01533143222332,
+      "epoch": 1.4939467312348669,
+      "grad_norm": 0.17201802134513855,
+      "learning_rate": 2.7648393442985403e-05,
+      "loss": 0.9994634389877319,
+      "mean_token_accuracy": 0.6991591900587082,
+      "num_tokens": 14965851.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.0423510372638702,
+      "epoch": 1.5036319612590798,
+      "grad_norm": 0.17345097661018372,
+      "learning_rate": 2.7384405073192454e-05,
+      "loss": 1.0389484167099,
+      "mean_token_accuracy": 0.6870414391160011,
+      "num_tokens": 15063279.0,
+      "step": 156
+    },
+    {
+      "entropy": 1.0268240571022034,
+      "epoch": 1.513317191283293,
+      "grad_norm": 0.18508543074131012,
+      "learning_rate": 2.7120148111887732e-05,
+      "loss": 1.029033899307251,
+      "mean_token_accuracy": 0.6905829161405563,
+      "num_tokens": 15160245.0,
+      "step": 157
+    },
+    {
+      "entropy": 1.0137058570981026,
+      "epoch": 1.5230024213075062,
+      "grad_norm": 0.1771853119134903,
+      "learning_rate": 2.6855652326319063e-05,
+      "loss": 1.011698603630066,
+      "mean_token_accuracy": 0.6954887434840202,
+      "num_tokens": 15257132.0,
+      "step": 158
+    },
+    {
+      "entropy": 1.0114770829677582,
+      "epoch": 1.5326876513317191,
+      "grad_norm": 0.1829320192337036,
+      "learning_rate": 2.659094751063666e-05,
+      "loss": 1.0108840465545654,
+      "mean_token_accuracy": 0.6935675665736198,
+      "num_tokens": 15354561.0,
+      "step": 159
+    },
+    {
+      "entropy": 1.0002705082297325,
+      "epoch": 1.542372881355932,
+      "grad_norm": 0.17417006194591522,
+      "learning_rate": 2.6326063482536934e-05,
+      "loss": 0.9976310729980469,
+      "mean_token_accuracy": 0.6973941773176193,
+      "num_tokens": 15451774.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.0380558297038078,
+      "epoch": 1.5520581113801453,
+      "grad_norm": 0.18505249917507172,
+      "learning_rate": 2.606103007990371e-05,
+      "loss": 1.027144193649292,
+      "mean_token_accuracy": 0.6899437606334686,
+      "num_tokens": 15545203.0,
+      "step": 161
+    },
+    {
+      "entropy": 1.0247594714164734,
+      "epoch": 1.5617433414043584,
+      "grad_norm": 0.17371951043605804,
+      "learning_rate": 2.579587715744712e-05,
+      "loss": 1.0144855976104736,
+      "mean_token_accuracy": 0.6931553333997726,
+      "num_tokens": 15642907.0,
+      "step": 162
+    },
+    {
+      "entropy": 1.025797888636589,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 0.17915304005146027,
+      "learning_rate": 2.5530634583340592e-05,
+      "loss": 1.014986276626587,
+      "mean_token_accuracy": 0.6926708817481995,
+      "num_tokens": 15739378.0,
+      "step": 163
+    },
+    {
+      "entropy": 1.0104742795228958,
+      "epoch": 1.5811138014527844,
+      "grad_norm": 0.17889732122421265,
+      "learning_rate": 2.526533223585641e-05,
+      "loss": 1.0031661987304688,
+      "mean_token_accuracy": 0.6982422098517418,
+      "num_tokens": 15836007.0,
+      "step": 164
+    },
+    {
+      "entropy": 1.0167404934763908,
+      "epoch": 1.5907990314769975,
+      "grad_norm": 0.1768142729997635,
+      "learning_rate": 2.5e-05,
+      "loss": 1.009070634841919,
+      "mean_token_accuracy": 0.6938488930463791,
+      "num_tokens": 15933286.0,
+      "step": 165
+    },
+    {
+      "entropy": 0.9942386075854301,
+      "epoch": 1.6004842615012107,
+      "grad_norm": 0.2674974203109741,
+      "learning_rate": 2.47346677641436e-05,
+      "loss": 0.9868994951248169,
+      "mean_token_accuracy": 0.7019253075122833,
+      "num_tokens": 16030240.0,
+      "step": 166
+    },
+    {
+      "entropy": 1.0080910325050354,
+      "epoch": 1.6101694915254239,
+      "grad_norm": 0.17362943291664124,
+      "learning_rate": 2.446936541665941e-05,
+      "loss": 1.0053434371948242,
+      "mean_token_accuracy": 0.6974755153059959,
+      "num_tokens": 16127744.0,
+      "step": 167
+    },
+    {
+      "entropy": 1.0292630195617676,
+      "epoch": 1.6198547215496368,
+      "grad_norm": 0.18086425960063934,
+      "learning_rate": 2.4204122842552896e-05,
+      "loss": 1.0237219333648682,
+      "mean_token_accuracy": 0.69173414260149,
+      "num_tokens": 16225217.0,
+      "step": 168
+    },
+    {
+      "entropy": 1.0308879762887955,
+      "epoch": 1.6295399515738498,
+      "grad_norm": 0.18374203145503998,
+      "learning_rate": 2.39389699200963e-05,
+      "loss": 1.0210704803466797,
+      "mean_token_accuracy": 0.6903956010937691,
+      "num_tokens": 16322913.0,
+      "step": 169
+    },
+    {
+      "entropy": 1.026255913078785,
+      "epoch": 1.639225181598063,
+      "grad_norm": 0.18278539180755615,
+      "learning_rate": 2.3673936517463075e-05,
+      "loss": 1.0169858932495117,
+      "mean_token_accuracy": 0.6928274035453796,
+      "num_tokens": 16419439.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.9919314235448837,
+      "epoch": 1.6489104116222761,
+      "grad_norm": 0.17901423573493958,
+      "learning_rate": 2.3409052489363342e-05,
+      "loss": 0.9918413162231445,
+      "mean_token_accuracy": 0.6992172300815582,
+      "num_tokens": 16516410.0,
+      "step": 171
+    },
+    {
+      "entropy": 1.0355833768844604,
+      "epoch": 1.658595641646489,
+      "grad_norm": 0.17641407251358032,
+      "learning_rate": 2.3144347673680936e-05,
+      "loss": 1.0381958484649658,
+      "mean_token_accuracy": 0.686020016670227,
+      "num_tokens": 16614080.0,
+      "step": 172
+    },
+    {
+      "entropy": 1.0149222612380981,
+      "epoch": 1.668280871670702,
+      "grad_norm": 0.17820627987384796,
+      "learning_rate": 2.287985188811228e-05,
+      "loss": 1.0130999088287354,
+      "mean_token_accuracy": 0.6923095658421516,
+      "num_tokens": 16711804.0,
+      "step": 173
+    },
+    {
+      "entropy": 1.0243929624557495,
+      "epoch": 1.6779661016949152,
+      "grad_norm": 0.18051660060882568,
+      "learning_rate": 2.261559492680755e-05,
+      "loss": 1.0211886167526245,
+      "mean_token_accuracy": 0.6929004341363907,
+      "num_tokens": 16808390.0,
+      "step": 174
+    },
+    {
+      "entropy": 1.008925937116146,
+      "epoch": 1.6876513317191284,
+      "grad_norm": 0.17846113443374634,
+      "learning_rate": 2.23516065570146e-05,
+      "loss": 0.9855537414550781,
+      "mean_token_accuracy": 0.7024274170398712,
+      "num_tokens": 16905846.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.0306098610162735,
+      "epoch": 1.6973365617433414,
+      "grad_norm": 0.17899896204471588,
+      "learning_rate": 2.2087916515725737e-05,
+      "loss": 1.0127032995224,
+      "mean_token_accuracy": 0.6937521025538445,
+      "num_tokens": 17002576.0,
+      "step": 176
+    },
+    {
+      "entropy": 1.0314870774745941,
+      "epoch": 1.7070217917675545,
+      "grad_norm": 0.1775345355272293,
+      "learning_rate": 2.182455450632803e-05,
+      "loss": 1.014747142791748,
+      "mean_token_accuracy": 0.6926901116967201,
+      "num_tokens": 17099808.0,
+      "step": 177
+    },
+    {
+      "entropy": 0.988661415874958,
+      "epoch": 1.7167070217917675,
+      "grad_norm": 0.18358361721038818,
+      "learning_rate": 2.1561550195257378e-05,
+      "loss": 0.9753472805023193,
+      "mean_token_accuracy": 0.7034848108887672,
+      "num_tokens": 17197523.0,
+      "step": 178
+    },
+    {
+      "entropy": 1.0072592198848724,
+      "epoch": 1.7263922518159807,
+      "grad_norm": 0.18197070062160492,
+      "learning_rate": 2.1298933208656718e-05,
+      "loss": 1.0025818347930908,
+      "mean_token_accuracy": 0.6967148259282112,
+      "num_tokens": 17292582.0,
+      "step": 179
+    },
+    {
+      "entropy": 1.0190014764666557,
+      "epoch": 1.7360774818401938,
+      "grad_norm": 0.1827293038368225,
+      "learning_rate": 2.103673312903878e-05,
+      "loss": 1.025465726852417,
+      "mean_token_accuracy": 0.691127598285675,
+      "num_tokens": 17389678.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.0355056896805763,
+      "epoch": 1.7457627118644068,
+      "grad_norm": 0.19289104640483856,
+      "learning_rate": 2.0774979491953777e-05,
+      "loss": 1.030361533164978,
+      "mean_token_accuracy": 0.6898203268647194,
+      "num_tokens": 17484545.0,
+      "step": 181
+    },
+    {
+      "entropy": 1.0205051228404045,
+      "epoch": 1.7554479418886197,
+      "grad_norm": 0.18466636538505554,
+      "learning_rate": 2.0513701782662366e-05,
+      "loss": 1.01216721534729,
+      "mean_token_accuracy": 0.6929588541388512,
+      "num_tokens": 17579500.0,
+      "step": 182
+    },
+    {
+      "entropy": 0.9954008832573891,
+      "epoch": 1.765133171912833,
+      "grad_norm": 0.18636250495910645,
+      "learning_rate": 2.025292943281429e-05,
+      "loss": 0.9910628199577332,
+      "mean_token_accuracy": 0.6983159705996513,
+      "num_tokens": 17674413.0,
+      "step": 183
+    },
+    {
+      "entropy": 1.00897516310215,
+      "epoch": 1.774818401937046,
+      "grad_norm": 0.1883116513490677,
+      "learning_rate": 1.9992691817133024e-05,
+      "loss": 1.0008057355880737,
+      "mean_token_accuracy": 0.6949929445981979,
+      "num_tokens": 17772206.0,
+      "step": 184
+    },
+    {
+      "entropy": 1.0539861172437668,
+      "epoch": 1.784503631961259,
+      "grad_norm": 0.188166081905365,
+      "learning_rate": 1.973301825010685e-05,
+      "loss": 1.0534381866455078,
+      "mean_token_accuracy": 0.6841985285282135,
+      "num_tokens": 17869518.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.0237281695008278,
+      "epoch": 1.794188861985472,
+      "grad_norm": 0.18438467383384705,
+      "learning_rate": 1.947393798268676e-05,
+      "loss": 1.0138177871704102,
+      "mean_token_accuracy": 0.6940690726041794,
+      "num_tokens": 17965115.0,
+      "step": 186
+    },
+    {
+      "entropy": 1.0318024158477783,
+      "epoch": 1.8038740920096852,
+      "grad_norm": 0.17989851534366608,
+      "learning_rate": 1.9215480198991466e-05,
+      "loss": 1.0178849697113037,
+      "mean_token_accuracy": 0.6889730393886566,
+      "num_tokens": 18062515.0,
+      "step": 187
+    },
+    {
+      "entropy": 1.0370707884430885,
+      "epoch": 1.8135593220338984,
+      "grad_norm": 0.18175791203975677,
+      "learning_rate": 1.895767401301994e-05,
+      "loss": 1.0233769416809082,
+      "mean_token_accuracy": 0.689442440867424,
+      "num_tokens": 18159901.0,
+      "step": 188
+    },
+    {
+      "entropy": 1.0387394651770592,
+      "epoch": 1.8232445520581115,
+      "grad_norm": 0.1763962209224701,
+      "learning_rate": 1.8700548465371874e-05,
+      "loss": 1.0317208766937256,
+      "mean_token_accuracy": 0.6874615550041199,
+      "num_tokens": 18257746.0,
+      "step": 189
+    },
+    {
+      "entropy": 1.016056202352047,
+      "epoch": 1.8329297820823245,
+      "grad_norm": 0.1811491698026657,
+      "learning_rate": 1.84441325199764e-05,
+      "loss": 1.0119330883026123,
+      "mean_token_accuracy": 0.6953575611114502,
+      "num_tokens": 18355948.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.9938200861215591,
+      "epoch": 1.8426150121065374,
+      "grad_norm": 0.1725209653377533,
+      "learning_rate": 1.818845506082943e-05,
+      "loss": 0.9812361598014832,
+      "mean_token_accuracy": 0.7028805017471313,
+      "num_tokens": 18453527.0,
+      "step": 191
+    },
+    {
+      "entropy": 1.031374305486679,
+      "epoch": 1.8523002421307506,
+      "grad_norm": 0.18246252834796906,
+      "learning_rate": 1.793354488874006e-05,
+      "loss": 1.0203440189361572,
+      "mean_token_accuracy": 0.6907096654176712,
+      "num_tokens": 18549884.0,
+      "step": 192
+    },
+    {
+      "entropy": 0.9885724261403084,
+      "epoch": 1.8619854721549638,
+      "grad_norm": 0.1804785132408142,
+      "learning_rate": 1.7679430718086243e-05,
+      "loss": 0.9796859622001648,
+      "mean_token_accuracy": 0.702710434794426,
+      "num_tokens": 18647122.0,
+      "step": 193
+    },
+    {
+      "entropy": 0.9969891235232353,
+      "epoch": 1.8716707021791767,
+      "grad_norm": 0.2115977555513382,
+      "learning_rate": 1.742614117358029e-05,
+      "loss": 0.9887296557426453,
+      "mean_token_accuracy": 0.6984992548823357,
+      "num_tokens": 18743129.0,
+      "step": 194
+    },
+    {
+      "entropy": 0.9905409216880798,
+      "epoch": 1.8813559322033897,
+      "grad_norm": 0.18363232910633087,
+      "learning_rate": 1.7173704787044446e-05,
+      "loss": 0.9921536445617676,
+      "mean_token_accuracy": 0.697743609547615,
+      "num_tokens": 18841235.0,
+      "step": 195
+    },
+    {
+      "entropy": 0.9967489168047905,
+      "epoch": 1.8910411622276029,
+      "grad_norm": 0.1787017434835434,
+      "learning_rate": 1.6922149994196878e-05,
+      "loss": 1.0015537738800049,
+      "mean_token_accuracy": 0.6942943632602692,
+      "num_tokens": 18938405.0,
+      "step": 196
+    },
+    {
+      "entropy": 1.017027884721756,
+      "epoch": 1.900726392251816,
+      "grad_norm": 0.1760513335466385,
+      "learning_rate": 1.667150513144856e-05,
+      "loss": 1.0226783752441406,
+      "mean_token_accuracy": 0.691545270383358,
+      "num_tokens": 19036103.0,
+      "step": 197
+    },
+    {
+      "entropy": 1.0095921084284782,
+      "epoch": 1.910411622276029,
+      "grad_norm": 0.1807909458875656,
+      "learning_rate": 1.6421798432711345e-05,
+      "loss": 1.004764437675476,
+      "mean_token_accuracy": 0.6964360028505325,
+      "num_tokens": 19131869.0,
+      "step": 198
+    },
+    {
+      "entropy": 1.0117909163236618,
+      "epoch": 1.9200968523002422,
+      "grad_norm": 0.18069379031658173,
+      "learning_rate": 1.617305802621748e-05,
+      "loss": 0.9964424967765808,
+      "mean_token_accuracy": 0.6977855116128922,
+      "num_tokens": 19229332.0,
+      "step": 199
+    },
+    {
+      "entropy": 1.0152455270290375,
+      "epoch": 1.9297820823244551,
+      "grad_norm": 0.1750374287366867,
+      "learning_rate": 1.5925311931351172e-05,
+      "loss": 1.0085126161575317,
+      "mean_token_accuracy": 0.6957272365689278,
+      "num_tokens": 19326511.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.0058443173766136,
+      "epoch": 1.9394673123486683,
+      "grad_norm": 0.17271851003170013,
+      "learning_rate": 1.567858805549229e-05,
+      "loss": 0.9887601733207703,
+      "mean_token_accuracy": 0.6980008259415627,
+      "num_tokens": 19423560.0,
+      "step": 201
+    },
+    {
+      "entropy": 1.0228681191802025,
+      "epoch": 1.9491525423728815,
+      "grad_norm": 0.17884734272956848,
+      "learning_rate": 1.5432914190872757e-05,
+      "loss": 1.0131609439849854,
+      "mean_token_accuracy": 0.692680798470974,
+      "num_tokens": 19520842.0,
+      "step": 202
+    },
+    {
+      "entropy": 1.0089232549071312,
+      "epoch": 1.9588377723970944,
+      "grad_norm": 0.16985908150672913,
+      "learning_rate": 1.5188318011445906e-05,
+      "loss": 1.0013556480407715,
+      "mean_token_accuracy": 0.6970010101795197,
+      "num_tokens": 19618823.0,
+      "step": 203
+    },
+    {
+      "entropy": 1.0016749277710915,
+      "epoch": 1.9685230024213074,
+      "grad_norm": 0.1778869777917862,
+      "learning_rate": 1.4944827069769123e-05,
+      "loss": 0.9919571876525879,
+      "mean_token_accuracy": 0.7001852169632912,
+      "num_tokens": 19716713.0,
+      "step": 204
+    },
+    {
+      "entropy": 1.0164751634001732,
+      "epoch": 1.9782082324455206,
+      "grad_norm": 0.18251122534275055,
+      "learning_rate": 1.4702468793900188e-05,
+      "loss": 1.0077866315841675,
+      "mean_token_accuracy": 0.6937248483300209,
+      "num_tokens": 19813324.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.0116839110851288,
+      "epoch": 1.9878934624697338,
+      "grad_norm": 0.17308309674263,
+      "learning_rate": 1.4461270484307643e-05,
+      "loss": 1.0083445310592651,
+      "mean_token_accuracy": 0.6930299252271652,
+      "num_tokens": 19910449.0,
+      "step": 206
+    },
+    {
+      "entropy": 1.0046918392181396,
+      "epoch": 1.9975786924939467,
+      "grad_norm": 0.17221423983573914,
+      "learning_rate": 1.4221259310795543e-05,
+      "loss": 0.9961509704589844,
+      "mean_token_accuracy": 0.6965227276086807,
+      "num_tokens": 20008513.0,
+      "step": 207
+    },
+    {
+      "entropy": 1.0230849385261536,
+      "epoch": 2.0,
+      "grad_norm": 0.35064876079559326,
+      "learning_rate": 1.3982462309442889e-05,
+      "loss": 1.0117878913879395,
+      "mean_token_accuracy": 0.6928033828735352,
+      "num_tokens": 20032830.0,
+      "step": 208
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.817820314388398e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}