Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-70B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-70B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-70B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:435ac09c3cf831d702d6dab1f1a4b66316fc2f3100748e5d6be5b59c79ce372c
+size 3313653480

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,1074 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 104,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2831476479768753,
+      "epoch": 0.009685230024213076,
+      "grad_norm": 1.257500410079956,
+      "learning_rate": 0.0,
+      "loss": 2.0216622352600098,
+      "mean_token_accuracy": 0.5382094085216522,
+      "num_tokens": 97485.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.2757050544023514,
+      "epoch": 0.01937046004842615,
+      "grad_norm": 1.3158080577850342,
+      "learning_rate": 3.125e-06,
+      "loss": 2.071122646331787,
+      "mean_token_accuracy": 0.5314131379127502,
+      "num_tokens": 194184.0,
+      "step": 2
+    },
+    {
+      "entropy": 1.2694316655397415,
+      "epoch": 0.029055690072639227,
+      "grad_norm": 1.1289395093917847,
+      "learning_rate": 6.25e-06,
+      "loss": 1.9294486045837402,
+      "mean_token_accuracy": 0.5518109127879143,
+      "num_tokens": 287660.0,
+      "step": 3
+    },
+    {
+      "entropy": 1.3956594914197922,
+      "epoch": 0.0387409200968523,
+      "grad_norm": 0.9914306998252869,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 1.941465139389038,
+      "mean_token_accuracy": 0.54258743673563,
+      "num_tokens": 384672.0,
+      "step": 4
+    },
+    {
+      "entropy": 1.3979678750038147,
+      "epoch": 0.048426150121065374,
+      "grad_norm": 0.7685282230377197,
+      "learning_rate": 1.25e-05,
+      "loss": 1.8111070394515991,
+      "mean_token_accuracy": 0.5647609829902649,
+      "num_tokens": 481921.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.6170997768640518,
+      "epoch": 0.05811138014527845,
+      "grad_norm": 0.5168716311454773,
+      "learning_rate": 1.5625e-05,
+      "loss": 1.7966232299804688,
+      "mean_token_accuracy": 0.5577141717076302,
+      "num_tokens": 577866.0,
+      "step": 6
+    },
+    {
+      "entropy": 1.6874641478061676,
+      "epoch": 0.06779661016949153,
+      "grad_norm": 0.4072948098182678,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 1.708348035812378,
+      "mean_token_accuracy": 0.5744073167443275,
+      "num_tokens": 674698.0,
+      "step": 7
+    },
+    {
+      "entropy": 1.7709944546222687,
+      "epoch": 0.0774818401937046,
+      "grad_norm": 0.4375021159648895,
+      "learning_rate": 2.1875e-05,
+      "loss": 1.6493638753890991,
+      "mean_token_accuracy": 0.5816963836550713,
+      "num_tokens": 771175.0,
+      "step": 8
+    },
+    {
+      "entropy": 1.869639590382576,
+      "epoch": 0.08716707021791767,
+      "grad_norm": 0.5281513333320618,
+      "learning_rate": 2.5e-05,
+      "loss": 1.6473325490951538,
+      "mean_token_accuracy": 0.5845584571361542,
+      "num_tokens": 868470.0,
+      "step": 9
+    },
+    {
+      "entropy": 1.8225883692502975,
+      "epoch": 0.09685230024213075,
+      "grad_norm": 0.5096434354782104,
+      "learning_rate": 2.8125000000000003e-05,
+      "loss": 1.590818166732788,
+      "mean_token_accuracy": 0.5921354293823242,
+      "num_tokens": 964784.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.7066199630498886,
+      "epoch": 0.10653753026634383,
+      "grad_norm": 0.4193650186061859,
+      "learning_rate": 3.125e-05,
+      "loss": 1.529557228088379,
+      "mean_token_accuracy": 0.5996061563491821,
+      "num_tokens": 1061836.0,
+      "step": 11
+    },
+    {
+      "entropy": 1.6335420161485672,
+      "epoch": 0.1162227602905569,
+      "grad_norm": 0.34470972418785095,
+      "learning_rate": 3.4375e-05,
+      "loss": 1.5362547636032104,
+      "mean_token_accuracy": 0.5929826945066452,
+      "num_tokens": 1158471.0,
+      "step": 12
+    },
+    {
+      "entropy": 1.4438551515340805,
+      "epoch": 0.12590799031476999,
+      "grad_norm": 0.27761033177375793,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.4457957744598389,
+      "mean_token_accuracy": 0.6143342331051826,
+      "num_tokens": 1255084.0,
+      "step": 13
+    },
+    {
+      "entropy": 1.392322763800621,
+      "epoch": 0.13559322033898305,
+      "grad_norm": 0.2844429016113281,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 1.4511373043060303,
+      "mean_token_accuracy": 0.6104418113827705,
+      "num_tokens": 1352697.0,
+      "step": 14
+    },
+    {
+      "entropy": 1.3371251970529556,
+      "epoch": 0.14527845036319612,
+      "grad_norm": 0.2822510898113251,
+      "learning_rate": 4.375e-05,
+      "loss": 1.4322829246520996,
+      "mean_token_accuracy": 0.6155784726142883,
+      "num_tokens": 1450346.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.2996839582920074,
+      "epoch": 0.1549636803874092,
+      "grad_norm": 0.2731319069862366,
+      "learning_rate": 4.6875e-05,
+      "loss": 1.399651288986206,
+      "mean_token_accuracy": 0.6181296408176422,
+      "num_tokens": 1546056.0,
+      "step": 16
+    },
+    {
+      "entropy": 1.286782756447792,
+      "epoch": 0.16464891041162227,
+      "grad_norm": 0.2287229746580124,
+      "learning_rate": 5e-05,
+      "loss": 1.3414286375045776,
+      "mean_token_accuracy": 0.6305811852216721,
+      "num_tokens": 1644072.0,
+      "step": 17
+    },
+    {
+      "entropy": 1.2884749919176102,
+      "epoch": 0.17433414043583534,
+      "grad_norm": 0.24129998683929443,
+      "learning_rate": 4.999859193643945e-05,
+      "loss": 1.2895803451538086,
+      "mean_token_accuracy": 0.6411498486995697,
+      "num_tokens": 1740823.0,
+      "step": 18
+    },
+    {
+      "entropy": 1.399221807718277,
+      "epoch": 0.18401937046004843,
+      "grad_norm": 0.24152599275112152,
+      "learning_rate": 4.999436790436924e-05,
+      "loss": 1.3620624542236328,
+      "mean_token_accuracy": 0.6247868910431862,
+      "num_tokens": 1837872.0,
+      "step": 19
+    },
+    {
+      "entropy": 1.3781700730323792,
+      "epoch": 0.1937046004842615,
+      "grad_norm": 0.22794979810714722,
+      "learning_rate": 4.9987328379605816e-05,
+      "loss": 1.3193318843841553,
+      "mean_token_accuracy": 0.6357730701565742,
+      "num_tokens": 1935638.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.3448941111564636,
+      "epoch": 0.2033898305084746,
+      "grad_norm": 0.20295539498329163,
+      "learning_rate": 4.9977474155117045e-05,
+      "loss": 1.2904666662216187,
+      "mean_token_accuracy": 0.6384864747524261,
+      "num_tokens": 2033151.0,
+      "step": 21
+    },
+    {
+      "entropy": 1.3269115835428238,
+      "epoch": 0.21307506053268765,
+      "grad_norm": 0.19907879829406738,
+      "learning_rate": 4.9964806340932865e-05,
+      "loss": 1.2992527484893799,
+      "mean_token_accuracy": 0.6387153342366219,
+      "num_tokens": 2130350.0,
+      "step": 22
+    },
+    {
+      "entropy": 1.2736097276210785,
+      "epoch": 0.22276029055690072,
+      "grad_norm": 0.1993221640586853,
+      "learning_rate": 4.9949326364020314e-05,
+      "loss": 1.2424592971801758,
+      "mean_token_accuracy": 0.6515100449323654,
+      "num_tokens": 2226505.0,
+      "step": 23
+    },
+    {
+      "entropy": 1.2606256753206253,
+      "epoch": 0.2324455205811138,
+      "grad_norm": 0.1928444802761078,
+      "learning_rate": 4.993103596812268e-05,
+      "loss": 1.2370142936706543,
+      "mean_token_accuracy": 0.6512897834181786,
+      "num_tokens": 2324337.0,
+      "step": 24
+    },
+    {
+      "entropy": 1.1828058958053589,
+      "epoch": 0.24213075060532688,
+      "grad_norm": 0.1695406287908554,
+      "learning_rate": 4.9909937213563165e-05,
+      "loss": 1.1700199842453003,
+      "mean_token_accuracy": 0.6674213483929634,
+      "num_tokens": 2419418.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.2690571248531342,
+      "epoch": 0.25181598062953997,
+      "grad_norm": 0.17888334393501282,
+      "learning_rate": 4.988603247701277e-05,
+      "loss": 1.2402149438858032,
+      "mean_token_accuracy": 0.6481731608510017,
+      "num_tokens": 2515666.0,
+      "step": 26
+    },
+    {
+      "entropy": 1.2278835028409958,
+      "epoch": 0.26150121065375304,
+      "grad_norm": 0.1878158003091812,
+      "learning_rate": 4.985932445122257e-05,
+      "loss": 1.2124648094177246,
+      "mean_token_accuracy": 0.6553028896450996,
+      "num_tokens": 2613197.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.2094718217849731,
+      "epoch": 0.2711864406779661,
+      "grad_norm": 0.1670471876859665,
+      "learning_rate": 4.982981614472039e-05,
+      "loss": 1.2036645412445068,
+      "mean_token_accuracy": 0.6590213552117348,
+      "num_tokens": 2709760.0,
+      "step": 28
+    },
+    {
+      "entropy": 1.1803771257400513,
+      "epoch": 0.28087167070217917,
+      "grad_norm": 0.16492998600006104,
+      "learning_rate": 4.979751088147192e-05,
+      "loss": 1.1872878074645996,
+      "mean_token_accuracy": 0.661787860095501,
+      "num_tokens": 2807308.0,
+      "step": 29
+    },
+    {
+      "entropy": 1.170692041516304,
+      "epoch": 0.29055690072639223,
+      "grad_norm": 0.17398205399513245,
+      "learning_rate": 4.97624123005063e-05,
+      "loss": 1.1832412481307983,
+      "mean_token_accuracy": 0.661918580532074,
+      "num_tokens": 2904379.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.1982254832983017,
+      "epoch": 0.30024213075060535,
+      "grad_norm": 0.16584837436676025,
+      "learning_rate": 4.972452435550614e-05,
+      "loss": 1.2053539752960205,
+      "mean_token_accuracy": 0.6568507477641106,
+      "num_tokens": 3001932.0,
+      "step": 31
+    },
+    {
+      "entropy": 1.139011174440384,
+      "epoch": 0.3099273607748184,
+      "grad_norm": 0.16256465017795563,
+      "learning_rate": 4.968385131436222e-05,
+      "loss": 1.1371057033538818,
+      "mean_token_accuracy": 0.6699384972453117,
+      "num_tokens": 3099859.0,
+      "step": 32
+    },
+    {
+      "entropy": 1.1760681718587875,
+      "epoch": 0.3196125907990315,
+      "grad_norm": 0.15591397881507874,
+      "learning_rate": 4.9640397758692715e-05,
+      "loss": 1.160001277923584,
+      "mean_token_accuracy": 0.6634443625807762,
+      "num_tokens": 3197294.0,
+      "step": 33
+    },
+    {
+      "entropy": 1.1871805638074875,
+      "epoch": 0.32929782082324455,
+      "grad_norm": 0.1550437957048416,
+      "learning_rate": 4.9594168583327094e-05,
+      "loss": 1.1692184209823608,
+      "mean_token_accuracy": 0.6639450341463089,
+      "num_tokens": 3294768.0,
+      "step": 34
+    },
+    {
+      "entropy": 1.1860681176185608,
+      "epoch": 0.3389830508474576,
+      "grad_norm": 0.15519270300865173,
+      "learning_rate": 4.954516899575473e-05,
+      "loss": 1.1605687141418457,
+      "mean_token_accuracy": 0.6660626903176308,
+      "num_tokens": 3389702.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.144161432981491,
+      "epoch": 0.3486682808716707,
+      "grad_norm": 0.16386644542217255,
+      "learning_rate": 4.949340451553833e-05,
+      "loss": 1.1134624481201172,
+      "mean_token_accuracy": 0.6774730086326599,
+      "num_tokens": 3486985.0,
+      "step": 36
+    },
+    {
+      "entropy": 1.169129118323326,
+      "epoch": 0.3583535108958838,
+      "grad_norm": 0.15737073123455048,
+      "learning_rate": 4.943888097369216e-05,
+      "loss": 1.1533232927322388,
+      "mean_token_accuracy": 0.6671040058135986,
+      "num_tokens": 3584177.0,
+      "step": 37
+    },
+    {
+      "entropy": 1.1375333964824677,
+      "epoch": 0.36803874092009686,
+      "grad_norm": 0.16654150187969208,
+      "learning_rate": 4.938160451202521e-05,
+      "loss": 1.1330571174621582,
+      "mean_token_accuracy": 0.6710135564208031,
+      "num_tokens": 3681733.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.115285873413086,
+      "epoch": 0.37772397094430993,
+      "grad_norm": 0.15554173290729523,
+      "learning_rate": 4.9321581582449365e-05,
+      "loss": 1.1185455322265625,
+      "mean_token_accuracy": 0.6746590584516525,
+      "num_tokens": 3777606.0,
+      "step": 39
+    },
+    {
+      "entropy": 1.1352528482675552,
+      "epoch": 0.387409200968523,
+      "grad_norm": 0.15042954683303833,
+      "learning_rate": 4.9258818946252624e-05,
+      "loss": 1.125117540359497,
+      "mean_token_accuracy": 0.6720233038067818,
+      "num_tokens": 3874534.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.1261803954839706,
+      "epoch": 0.39709443099273606,
+      "grad_norm": 0.1618603765964508,
+      "learning_rate": 4.9193323673337476e-05,
+      "loss": 1.1155072450637817,
+      "mean_token_accuracy": 0.6748175323009491,
+      "num_tokens": 3972562.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.1054729223251343,
+      "epoch": 0.4067796610169492,
+      "grad_norm": 0.1928945779800415,
+      "learning_rate": 4.912510314142448e-05,
+      "loss": 1.0943855047225952,
+      "mean_token_accuracy": 0.6810023561120033,
+      "num_tokens": 4069253.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.1252695173025131,
+      "epoch": 0.41646489104116224,
+      "grad_norm": 0.16767306625843048,
+      "learning_rate": 4.9054165035221236e-05,
+      "loss": 1.1221925020217896,
+      "mean_token_accuracy": 0.6732571944594383,
+      "num_tokens": 4167434.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.0451640114188194,
+      "epoch": 0.4261501210653753,
+      "grad_norm": 0.15018437802791595,
+      "learning_rate": 4.898051734555676e-05,
+      "loss": 1.0456998348236084,
+      "mean_token_accuracy": 0.6917429268360138,
+      "num_tokens": 4263989.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.0243945717811584,
+      "epoch": 0.4358353510895884,
+      "grad_norm": 0.15813371539115906,
+      "learning_rate": 4.890416836848127e-05,
+      "loss": 1.019399642944336,
+      "mean_token_accuracy": 0.6970273479819298,
+      "num_tokens": 4358172.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.0881676077842712,
+      "epoch": 0.44552058111380144,
+      "grad_norm": 0.1647815704345703,
+      "learning_rate": 4.88251267043318e-05,
+      "loss": 1.089061975479126,
+      "mean_token_accuracy": 0.6811036169528961,
+      "num_tokens": 4454874.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.1170424669981003,
+      "epoch": 0.4552058111380145,
+      "grad_norm": 0.15906141698360443,
+      "learning_rate": 4.87434012567633e-05,
+      "loss": 1.1157368421554565,
+      "mean_token_accuracy": 0.6773378178477287,
+      "num_tokens": 4551716.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.1030111014842987,
+      "epoch": 0.4648910411622276,
+      "grad_norm": 0.1497914344072342,
+      "learning_rate": 4.865900123174574e-05,
+      "loss": 1.0889781713485718,
+      "mean_token_accuracy": 0.6786394864320755,
+      "num_tokens": 4649881.0,
+      "step": 48
+    },
+    {
+      "entropy": 1.1213176995515823,
+      "epoch": 0.4745762711864407,
+      "grad_norm": 0.14933931827545166,
+      "learning_rate": 4.857193613652711e-05,
+      "loss": 1.102480173110962,
+      "mean_token_accuracy": 0.6778694689273834,
+      "num_tokens": 4746560.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.1182024255394936,
+      "epoch": 0.48426150121065376,
+      "grad_norm": 0.1557791829109192,
+      "learning_rate": 4.848221577856243e-05,
+      "loss": 1.1001962423324585,
+      "mean_token_accuracy": 0.6782753691077232,
+      "num_tokens": 4844482.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.0782635807991028,
+      "epoch": 0.4939467312348668,
+      "grad_norm": 0.15607225894927979,
+      "learning_rate": 4.838985026440905e-05,
+      "loss": 1.0587979555130005,
+      "mean_token_accuracy": 0.6873368322849274,
+      "num_tokens": 4942340.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.071622334420681,
+      "epoch": 0.5036319612590799,
+      "grad_norm": 0.1505449265241623,
+      "learning_rate": 4.829484999858815e-05,
+      "loss": 1.0695254802703857,
+      "mean_token_accuracy": 0.6835980787873268,
+      "num_tokens": 5040296.0,
+      "step": 52
+    },
+    {
+      "entropy": 1.0763383507728577,
+      "epoch": 0.513317191283293,
+      "grad_norm": 0.15331462025642395,
+      "learning_rate": 4.819722568241274e-05,
+      "loss": 1.073240041732788,
+      "mean_token_accuracy": 0.6868264526128769,
+      "num_tokens": 5136985.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.086071863770485,
+      "epoch": 0.5230024213075061,
+      "grad_norm": 0.15281084179878235,
+      "learning_rate": 4.8096988312782174e-05,
+      "loss": 1.089070200920105,
+      "mean_token_accuracy": 0.6783855706453323,
+      "num_tokens": 5234555.0,
+      "step": 54
+    },
+    {
+      "entropy": 1.0728528648614883,
+      "epoch": 0.5326876513317191,
+      "grad_norm": 0.15144552290439606,
+      "learning_rate": 4.799414918094347e-05,
+      "loss": 1.0666192770004272,
+      "mean_token_accuracy": 0.6838826537132263,
+      "num_tokens": 5330627.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.0615920722484589,
+      "epoch": 0.5423728813559322,
+      "grad_norm": 0.25972187519073486,
+      "learning_rate": 4.788871987121937e-05,
+      "loss": 1.0613664388656616,
+      "mean_token_accuracy": 0.6867948919534683,
+      "num_tokens": 5428196.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.066960223019123,
+      "epoch": 0.5520581113801453,
+      "grad_norm": 0.15711794793605804,
+      "learning_rate": 4.77807122597034e-05,
+      "loss": 1.0630605220794678,
+      "mean_token_accuracy": 0.6852066963911057,
+      "num_tokens": 5525318.0,
+      "step": 57
+    },
+    {
+      "entropy": 1.0585280284285545,
+      "epoch": 0.5617433414043583,
+      "grad_norm": 0.154302716255188,
+      "learning_rate": 4.767013851292212e-05,
+      "loss": 1.048720359802246,
+      "mean_token_accuracy": 0.6897359937429428,
+      "num_tokens": 5623136.0,
+      "step": 58
+    },
+    {
+      "entropy": 1.0350576639175415,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.15237243473529816,
+      "learning_rate": 4.755701108646463e-05,
+      "loss": 1.027522325515747,
+      "mean_token_accuracy": 0.6950520128011703,
+      "num_tokens": 5720541.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.0580420941114426,
+      "epoch": 0.5811138014527845,
+      "grad_norm": 0.14978040754795074,
+      "learning_rate": 4.744134272357948e-05,
+      "loss": 1.0437177419662476,
+      "mean_token_accuracy": 0.6894178092479706,
+      "num_tokens": 5818052.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.047768495976925,
+      "epoch": 0.5907990314769975,
+      "grad_norm": 0.45630910992622375,
+      "learning_rate": 4.732314645373921e-05,
+      "loss": 1.0360010862350464,
+      "mean_token_accuracy": 0.6936607509851456,
+      "num_tokens": 5915984.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.0626960098743439,
+      "epoch": 0.6004842615012107,
+      "grad_norm": 0.2640538513660431,
+      "learning_rate": 4.7202435591172676e-05,
+      "loss": 1.0494108200073242,
+      "mean_token_accuracy": 0.6885051354765892,
+      "num_tokens": 6013024.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.0739953070878983,
+      "epoch": 0.6101694915254238,
+      "grad_norm": 0.15643136203289032,
+      "learning_rate": 4.7079223733365237e-05,
+      "loss": 1.0664867162704468,
+      "mean_token_accuracy": 0.6844003722071648,
+      "num_tokens": 6110253.0,
+      "step": 63
+    },
+    {
+      "entropy": 1.0349700078368187,
+      "epoch": 0.6198547215496368,
+      "grad_norm": 0.1503216028213501,
+      "learning_rate": 4.6953524759527054e-05,
+      "loss": 1.0348119735717773,
+      "mean_token_accuracy": 0.692238837480545,
+      "num_tokens": 6207767.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.034246288239956,
+      "epoch": 0.6295399515738499,
+      "grad_norm": 0.1513209342956543,
+      "learning_rate": 4.6825352829029705e-05,
+      "loss": 1.0328264236450195,
+      "mean_token_accuracy": 0.6916565299034119,
+      "num_tokens": 6305333.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.0430887341499329,
+      "epoch": 0.639225181598063,
+      "grad_norm": 0.15611153841018677,
+      "learning_rate": 4.6694722379811185e-05,
+      "loss": 1.0414214134216309,
+      "mean_token_accuracy": 0.691117949783802,
+      "num_tokens": 6402785.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.0569112747907639,
+      "epoch": 0.648910411622276,
+      "grad_norm": 0.1588103026151657,
+      "learning_rate": 4.656164812674951e-05,
+      "loss": 1.047569990158081,
+      "mean_token_accuracy": 0.6887474879622459,
+      "num_tokens": 6499919.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.0378888696432114,
+      "epoch": 0.6585956416464891,
+      "grad_norm": 0.15132373571395874,
+      "learning_rate": 4.642614506000523e-05,
+      "loss": 1.0328857898712158,
+      "mean_token_accuracy": 0.6924383491277695,
+      "num_tokens": 6597204.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.032283864915371,
+      "epoch": 0.6682808716707022,
+      "grad_norm": 0.1583699733018875,
+      "learning_rate": 4.628822844333278e-05,
+      "loss": 1.0151689052581787,
+      "mean_token_accuracy": 0.6986712738871574,
+      "num_tokens": 6694398.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.0015813261270523,
+      "epoch": 0.6779661016949152,
+      "grad_norm": 0.1757071167230606,
+      "learning_rate": 4.614791381236115e-05,
+      "loss": 0.9920505285263062,
+      "mean_token_accuracy": 0.7040144950151443,
+      "num_tokens": 6791780.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.0453337281942368,
+      "epoch": 0.6876513317191283,
+      "grad_norm": 0.1524701863527298,
+      "learning_rate": 4.6005216972843864e-05,
+      "loss": 1.0443840026855469,
+      "mean_token_accuracy": 0.6916699558496475,
+      "num_tokens": 6888953.0,
+      "step": 71
+    },
+    {
+      "entropy": 1.0230370834469795,
+      "epoch": 0.6973365617433414,
+      "grad_norm": 0.15993070602416992,
+      "learning_rate": 4.5860153998878494e-05,
+      "loss": 1.0304991006851196,
+      "mean_token_accuracy": 0.6937867403030396,
+      "num_tokens": 6986471.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.0421988740563393,
+      "epoch": 0.7070217917675545,
+      "grad_norm": 0.1573047935962677,
+      "learning_rate": 4.571274123109606e-05,
+      "loss": 1.0444982051849365,
+      "mean_token_accuracy": 0.6912301033735275,
+      "num_tokens": 7083829.0,
+      "step": 73
+    },
+    {
+      "entropy": 1.0748334154486656,
+      "epoch": 0.7167070217917676,
+      "grad_norm": 0.16008728742599487,
+      "learning_rate": 4.5562995274820284e-05,
+      "loss": 1.0726516246795654,
+      "mean_token_accuracy": 0.6823427230119705,
+      "num_tokens": 7180818.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.0194222182035446,
+      "epoch": 0.7263922518159807,
+      "grad_norm": 0.16299541294574738,
+      "learning_rate": 4.541093299819714e-05,
+      "loss": 1.003331184387207,
+      "mean_token_accuracy": 0.6996421813964844,
+      "num_tokens": 7277648.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.0004182010889053,
+      "epoch": 0.7360774818401937,
+      "grad_norm": 0.15372581779956818,
+      "learning_rate": 4.5256571530294664e-05,
+      "loss": 0.9760631918907166,
+      "mean_token_accuracy": 0.7064241617918015,
+      "num_tokens": 7375160.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.0313043221831322,
+      "epoch": 0.7457627118644068,
+      "grad_norm": 0.15582375228405,
+      "learning_rate": 4.5099928259173516e-05,
+      "loss": 1.0199785232543945,
+      "mean_token_accuracy": 0.6969870626926422,
+      "num_tokens": 7472481.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.0267394110560417,
+      "epoch": 0.7554479418886199,
+      "grad_norm": 0.15595640242099762,
+      "learning_rate": 4.4941020829928245e-05,
+      "loss": 1.0236713886260986,
+      "mean_token_accuracy": 0.697633869946003,
+      "num_tokens": 7569978.0,
+      "step": 78
+    },
+    {
+      "entropy": 0.9967087134718895,
+      "epoch": 0.7651331719128329,
+      "grad_norm": 0.15426741540431976,
+      "learning_rate": 4.477986714269972e-05,
+      "loss": 0.9990708231925964,
+      "mean_token_accuracy": 0.7013676688075066,
+      "num_tokens": 7668053.0,
+      "step": 79
+    },
+    {
+      "entropy": 0.9936242178082466,
+      "epoch": 0.774818401937046,
+      "grad_norm": 0.15127019584178925,
+      "learning_rate": 4.4616485350658685e-05,
+      "loss": 0.992855429649353,
+      "mean_token_accuracy": 0.7030186578631401,
+      "num_tokens": 7764243.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.0158754512667656,
+      "epoch": 0.784503631961259,
+      "grad_norm": 0.1566711664199829,
+      "learning_rate": 4.445089385796099e-05,
+      "loss": 1.0183484554290771,
+      "mean_token_accuracy": 0.6949350982904434,
+      "num_tokens": 7860199.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.035573497414589,
+      "epoch": 0.7941888619854721,
+      "grad_norm": 0.15846672654151917,
+      "learning_rate": 4.4283111317674374e-05,
+      "loss": 1.0358930826187134,
+      "mean_token_accuracy": 0.6919568032026291,
+      "num_tokens": 7956920.0,
+      "step": 82
+    },
+    {
+      "entropy": 1.0298198834061623,
+      "epoch": 0.8038740920096852,
+      "grad_norm": 0.1551441252231598,
+      "learning_rate": 4.4113156629677316e-05,
+      "loss": 1.02339768409729,
+      "mean_token_accuracy": 0.6937244087457657,
+      "num_tokens": 8053766.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.0543159544467926,
+      "epoch": 0.8135593220338984,
+      "grad_norm": 0.15082016587257385,
+      "learning_rate": 4.394104893853007e-05,
+      "loss": 1.0417697429656982,
+      "mean_token_accuracy": 0.6908006444573402,
+      "num_tokens": 8151263.0,
+      "step": 84
+    },
+    {
+      "entropy": 0.9868235737085342,
+      "epoch": 0.8232445520581114,
+      "grad_norm": 0.15088169276714325,
+      "learning_rate": 4.3766807631318106e-05,
+      "loss": 0.9730858206748962,
+      "mean_token_accuracy": 0.7068010121583939,
+      "num_tokens": 8249031.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.0439471304416656,
+      "epoch": 0.8329297820823245,
+      "grad_norm": 0.15690501034259796,
+      "learning_rate": 4.359045233546827e-05,
+      "loss": 1.033866286277771,
+      "mean_token_accuracy": 0.6926037818193436,
+      "num_tokens": 8345911.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.0188320130109787,
+      "epoch": 0.8426150121065376,
+      "grad_norm": 0.16062293946743011,
+      "learning_rate": 4.341200291653781e-05,
+      "loss": 1.0072400569915771,
+      "mean_token_accuracy": 0.7004816979169846,
+      "num_tokens": 8440245.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.0310387164354324,
+      "epoch": 0.8523002421307506,
+      "grad_norm": 0.15309439599514008,
+      "learning_rate": 4.323147947597666e-05,
+      "loss": 1.0136595964431763,
+      "mean_token_accuracy": 0.6977442428469658,
+      "num_tokens": 8537604.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.0147035270929337,
+      "epoch": 0.8619854721549637,
+      "grad_norm": 0.16039453446865082,
+      "learning_rate": 4.3048902348863116e-05,
+      "loss": 1.0053828954696655,
+      "mean_token_accuracy": 0.6996104419231415,
+      "num_tokens": 8634832.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.009638950228691,
+      "epoch": 0.8716707021791767,
+      "grad_norm": 0.15286219120025635,
+      "learning_rate": 4.2864292101613136e-05,
+      "loss": 1.012491226196289,
+      "mean_token_accuracy": 0.6957853958010674,
+      "num_tokens": 8732321.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.0169770643115044,
+      "epoch": 0.8813559322033898,
+      "grad_norm": 0.16297462582588196,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 1.0225484371185303,
+      "mean_token_accuracy": 0.6936622560024261,
+      "num_tokens": 8830140.0,
+      "step": 91
+    },
+    {
+      "entropy": 0.9886957630515099,
+      "epoch": 0.8910411622276029,
+      "grad_norm": 0.18117401003837585,
+      "learning_rate": 4.248905565513023e-05,
+      "loss": 0.9961055517196655,
+      "mean_token_accuracy": 0.699328251183033,
+      "num_tokens": 8926792.0,
+      "step": 92
+    },
+    {
+      "entropy": 1.026213452219963,
+      "epoch": 0.9007263922518159,
+      "grad_norm": 0.15571565926074982,
+      "learning_rate": 4.229847172443866e-05,
+      "loss": 1.0185178518295288,
+      "mean_token_accuracy": 0.6954792588949203,
+      "num_tokens": 9024094.0,
+      "step": 93
+    },
+    {
+      "entropy": 0.9805751740932465,
+      "epoch": 0.910411622276029,
+      "grad_norm": 0.1524103581905365,
+      "learning_rate": 4.210593920593201e-05,
+      "loss": 0.9706284999847412,
+      "mean_token_accuracy": 0.7079346030950546,
+      "num_tokens": 9120978.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.0263385400176048,
+      "epoch": 0.9200968523002422,
+      "grad_norm": 0.1659349501132965,
+      "learning_rate": 4.191147978745218e-05,
+      "loss": 1.010141134262085,
+      "mean_token_accuracy": 0.6972236707806587,
+      "num_tokens": 9217940.0,
+      "step": 95
+    },
+    {
+      "entropy": 0.9710419476032257,
+      "epoch": 0.9297820823244553,
+      "grad_norm": 0.14841604232788086,
+      "learning_rate": 4.171511537389684e-05,
+      "loss": 0.9553003311157227,
+      "mean_token_accuracy": 0.7146916389465332,
+      "num_tokens": 9315038.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.0402830019593239,
+      "epoch": 0.9394673123486683,
+      "grad_norm": 0.1693028062582016,
+      "learning_rate": 4.151686808475204e-05,
+      "loss": 1.022579312324524,
+      "mean_token_accuracy": 0.6941675096750259,
+      "num_tokens": 9411796.0,
+      "step": 97
+    },
+    {
+      "entropy": 0.9890827238559723,
+      "epoch": 0.9491525423728814,
+      "grad_norm": 0.15846097469329834,
+      "learning_rate": 4.131676025160047e-05,
+      "loss": 0.9882045984268188,
+      "mean_token_accuracy": 0.7028019949793816,
+      "num_tokens": 9508011.0,
+      "step": 98
+    },
+    {
+      "entropy": 0.964131772518158,
+      "epoch": 0.9588377723970944,
+      "grad_norm": 0.1603643000125885,
+      "learning_rate": 4.111481441560598e-05,
+      "loss": 0.9728220701217651,
+      "mean_token_accuracy": 0.7061599791049957,
+      "num_tokens": 9605557.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.0129969343543053,
+      "epoch": 0.9685230024213075,
+      "grad_norm": 0.1754368096590042,
+      "learning_rate": 4.091105332497439e-05,
+      "loss": 1.0291517972946167,
+      "mean_token_accuracy": 0.6916319876909256,
+      "num_tokens": 9703059.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.0249989926815033,
+      "epoch": 0.9782082324455206,
+      "grad_norm": 0.16457504034042358,
+      "learning_rate": 4.070549993239106e-05,
+      "loss": 1.0286558866500854,
+      "mean_token_accuracy": 0.6931352093815804,
+      "num_tokens": 9800187.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.0157797187566757,
+      "epoch": 0.9878934624697336,
+      "grad_norm": 0.1640177220106125,
+      "learning_rate": 4.049817739243532e-05,
+      "loss": 1.001893162727356,
+      "mean_token_accuracy": 0.6987328082323074,
+      "num_tokens": 9897311.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.0060301199555397,
+      "epoch": 0.9975786924939467,
+      "grad_norm": 0.15887762606143951,
+      "learning_rate": 4.028910905897229e-05,
+      "loss": 0.9999165534973145,
+      "mean_token_accuracy": 0.7007258981466293,
+      "num_tokens": 9994136.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.057703673839569,
+      "epoch": 1.0,
+      "grad_norm": 0.2889890670776367,
+      "learning_rate": 4.007831848252211e-05,
+      "loss": 1.043251395225525,
+      "mean_token_accuracy": 0.6889578104019165,
+      "num_tokens": 10018679.0,
+      "step": 104
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.4092583161015828e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}