Add 11 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training-metrics.png filter=lfs diff=lfs merge=lfs -text
+win-rates.png filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,102 @@
+---
+base_model: meta-llama/Llama-4-Scout-17B-16E-Instruct
+library_name: peft
+license: other
+tags:
+  - lora
+  - peft
+  - adapter
+  - adaption
+---
+
+# adaption_chartsense_sft
+
+## Model Training
+
+A LORA adapter for `meta-llama/Llama-4-Scout-17B-16E-Instruct`. This model was trained with SFT using [Adaption](https://adaptionlabs.ai)'s AutoScientist on the chartsense_sft dataset.
+
+
+![Training metrics](training-metrics.png)
+
+### AutoScientist Config
+
+```json
+{
+  "job_id": "1e548ea0-c1a0-4aeb-8563-f35fa19cd991",
+  "training_experiment_id": "d24efb7c-6ba7-4a94-910d-7596b10720c1",
+  "original_model_name": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+  "trained_model_name": "adaption_chartsense_sft",
+  "training_method": "sft",
+  "training_type": "lora",
+  "data_format": "chat",
+  "hyperparams": {
+    "lora": "true",
+    "lora_r": 64,
+    "n_evals": 5,
+    "n_epochs": 4,
+    "batch_size": "max",
+    "lora_alpha": 128,
+    "lora_dropout": 0,
+    "min_lr_ratio": 0.1,
+    "warmup_ratio": 0.03,
+    "weight_decay": 0.02,
+    "learning_rate": 0.0001,
+    "max_grad_norm": 1,
+    "base_model_size": "109B",
+    "train_on_inputs": "false",
+    "training_method": "sft",
+    "lr_scheduler_type": "cosine",
+    "scheduler_num_cycles": 0.5,
+    "lora_trainable_modules": "k_proj,o_proj,q_proj,v_proj,shared_expert.gate_proj,shared_expert.up_proj,shared_expert.down_proj,feed_forward.gate_proj,feed_forward.up_proj,feed_forward.down_proj"
+  }
+}
+```
+
+## Training Data
+
+The model was trained on 25,397 rows of adapted data with the following domain distribution: data-analysis-visualization (96%), code (2%), science (0%), academic-education (0%), corporate-business (0%), sports (0%), market-analysis (0%), math (0%), marketing (0%), games (0%), hr (0%), technology (0%), agriculture (0%).
+
+## Model Evaluation
+
+The model was evaluated on an in-distribution held-out test set as well as a broader domain-specific test set to measure generalization.
+
+
+![Win rates](win-rates.png)
+
+| Domain | Win rate vs. base model |
+| --- | --- |
+| data-analysis-visualization | 79% |
+
+## How to use
+
+```bash
+pip install torch transformers peft
+```
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+BASE = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
+ADAPTER = "<this-repo-id>"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.float32 if device == "cpu" else torch.bfloat16
+
+base = AutoModelForCausalLM.from_pretrained(BASE, dtype=dtype).to(device)
+model = PeftModel.from_pretrained(base, ADAPTER)
+# Optional: merge the LoRA weights into the base for faster inference
+model = model.merge_and_unload()
+model.eval()
+
+tokenizer = AutoTokenizer.from_pretrained(BASE)
+messages = [{"role": "user", "content": "Hello!"}]
+text = tokenizer.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True)
+inputs = tokenizer(text, return_tensors="pt").to(device)
+
+with torch.inference_mode():
+    out = model.generate(**inputs, max_new_tokens=512)
+print(tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+```

adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "togethercomputer/Llama-4-Scout-17B-16E-Instruct_bnb_4bit",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": [],
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "feed_forward.gate_proj",
+    "feed_forward.up_proj",
+    "q_proj",
+    "shared_expert.gate_proj",
+    "o_proj",
+    "k_proj",
+    "shared_expert.up_proj",
+    "v_proj",
+    "shared_expert.down_proj",
+    "feed_forward.down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a99b4833b44522a64fb9e2d89949a32063a2747672d5a66699b2af8dd1e88ad
+size 893484120

chat_template.jinja

@@ -0,0 +1,123 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}    
+    {%- if messages[0]['content'] is string %}
+        {%- set system_message = messages[0]['content']|trim %}
+    {%- else %}
+        {#- FIXME: The processor requires an array, always. #}
+        {%- set system_message = messages[0]['content'][0]['text']|trim %}
+    {%- endif %}
+    {%- set messages = messages[1:] %}
+    {%- set user_supplied_system_message = true %}
+{%- else %}
+    {%- set system_message = "" %}
+    {%- set user_supplied_system_message = false %}
+{%- endif %}
+
+{#- System message if the user supplied one #}
+{%- if user_supplied_system_message %}
+    {{- "<|header_start|>system<|header_end|>\n\n" }}
+    {%- if tools is not none %}
+        {{- "Environment: ipython\n" }}
+    {%- endif %}
+    {%- if tools is not none and not tools_in_user_message %}
+        {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+        {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+        {{- "Do not use variables.\n\n" }}
+        {%- for t in tools %}
+            {{- t | tojson(indent=4) }}
+            {{- "\n\n" }}
+        {%- endfor %}
+    {%- endif %}
+    {{- system_message }}
+    {{- "<|eot|>" }}
+{%- endif %}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|header_start|>user<|header_end|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+    {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
+        {%- if message['content'] is string %}
+            {{- message['content'] }}
+        {%- else %}
+            {%- for content in message['content'] %}
+                {%- if content['type'] == 'image' %}
+                    {{- '<|image|>' }}
+                {%- elif content['type'] == 'text' %}
+                    {{- content['text'] }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- "<|eot|>" }}
+    {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
+       {{- '<|header_start|>assistant<|header_end|>\n\n' -}}
+       {{- '<|python_start|>' }}
+        {%- if message['content'] is string %}
+            {{- message['content'] }}
+        {%- else %}
+            {%- for content in message['content'] %}
+                {%- if content['type'] == 'image' %}
+                    {{- '<|image|>' }}
+                {%- elif content['type'] == 'text' %}
+                    {{- content['text'] }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+       {{- '<|python_end|>' }}
+        {%- for tool_call in message.tool_calls %}
+           {{- '{"name": "' + tool_call.function.name + '", ' }}
+           {{- '"parameters": ' }}
+           {{- tool_call.function.arguments | tojson }}
+           {{- "}" }}
+        {%- endfor %}
+       {{- "<|eot|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|header_start|>ipython<|header_end|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|header_start|>assistant<|header_end|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,215 @@
+{
+  "architectures": [
+    "Llama4ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_chunk_size": 8192,
+  "attention_dropout": 0.0,
+  "attn_scale": 0.1,
+  "attn_temperature_tuning": true,
+  "bos_token_id": 200000,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 200008,
+  "floor_scale": 8192,
+  "for_llm_compressor": false,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "interleave_moe_layer_step": 1,
+  "intermediate_size": 8192,
+  "intermediate_size_mlp": 16384,
+  "layer_types": [
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 10485760,
+  "model_type": "llama4_text",
+  "moe_layers": [
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47
+  ],
+  "no_rope_layer_interval": 4,
+  "no_rope_layers": [
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0
+  ],
+  "num_attention_heads": 40,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "num_local_experts": 16,
+  "output_router_logits": false,
+  "pad_token_id": 200018,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 16.0,
+    "high_freq_factor": 1.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.13.0",
+  "use_cache": false,
+  "use_qk_norm": true,
+  "vocab_size": 202048,
+  "torch_dtype": "bfloat16"
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|eot|>",
+  "pad_token": "<|finetune_right_pad|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:172c9eb4beafc72601690da3ccfcede5c2e6806a8d5ec1fca33e22acea8023a4
+size 27948578

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|eot|>",
+  "is_local": false,
+  "local_files_only": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 10485760,
+  "pad_token": "<|finetune_right_pad|>",
+  "padding_side": "right",
+  "processor_class": "Llama4Processor",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,1726 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 47,
+  "global_step": 236,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01694915254237288,
+      "grad_norm": 0.6888771057128906,
+      "learning_rate": 0.0,
+      "loss": 1.2392578125,
+      "step": 1
+    },
+    {
+      "epoch": 0.03389830508474576,
+      "grad_norm": 0.866387128829956,
+      "learning_rate": 1.25e-05,
+      "loss": 1.498046875,
+      "step": 2
+    },
+    {
+      "epoch": 0.05084745762711865,
+      "grad_norm": 0.6771768927574158,
+      "learning_rate": 2.5e-05,
+      "loss": 1.28759765625,
+      "step": 3
+    },
+    {
+      "epoch": 0.06779661016949153,
+      "grad_norm": 0.6170372366905212,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.22265625,
+      "step": 4
+    },
+    {
+      "epoch": 0.0847457627118644,
+      "grad_norm": 0.5621561408042908,
+      "learning_rate": 5e-05,
+      "loss": 1.2998046875,
+      "step": 5
+    },
+    {
+      "epoch": 0.1016949152542373,
+      "grad_norm": 0.3775511085987091,
+      "learning_rate": 6.25e-05,
+      "loss": 1.28173828125,
+      "step": 6
+    },
+    {
+      "epoch": 0.11864406779661017,
+      "grad_norm": 0.2543378472328186,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 1.24755859375,
+      "step": 7
+    },
+    {
+      "epoch": 0.13559322033898305,
+      "grad_norm": 0.24368329346179962,
+      "learning_rate": 8.75e-05,
+      "loss": 1.1640625,
+      "step": 8
+    },
+    {
+      "epoch": 0.15254237288135594,
+      "grad_norm": 0.37695613503456116,
+      "learning_rate": 0.0001,
+      "loss": 1.037109375,
+      "step": 9
+    },
+    {
+      "epoch": 0.1694915254237288,
+      "grad_norm": 0.6395261883735657,
+      "learning_rate": 9.999572825127697e-05,
+      "loss": 1.20068359375,
+      "step": 10
+    },
+    {
+      "epoch": 0.1864406779661017,
+      "grad_norm": 0.3627767264842987,
+      "learning_rate": 9.998291381612281e-05,
+      "loss": 1.10791015625,
+      "step": 11
+    },
+    {
+      "epoch": 0.2033898305084746,
+      "grad_norm": 0.19022125005722046,
+      "learning_rate": 9.996155912742855e-05,
+      "loss": 0.840576171875,
+      "step": 12
+    },
+    {
+      "epoch": 0.22033898305084745,
+      "grad_norm": 0.1781640350818634,
+      "learning_rate": 9.993166823949923e-05,
+      "loss": 1.0634765625,
+      "step": 13
+    },
+    {
+      "epoch": 0.23728813559322035,
+      "grad_norm": 0.20156298577785492,
+      "learning_rate": 9.98932468272843e-05,
+      "loss": 1.17626953125,
+      "step": 14
+    },
+    {
+      "epoch": 0.2542372881355932,
+      "grad_norm": 0.17879654467105865,
+      "learning_rate": 9.984630218530014e-05,
+      "loss": 0.89794921875,
+      "step": 15
+    },
+    {
+      "epoch": 0.2711864406779661,
+      "grad_norm": 0.18100528419017792,
+      "learning_rate": 9.979084322624518e-05,
+      "loss": 1.15283203125,
+      "step": 16
+    },
+    {
+      "epoch": 0.288135593220339,
+      "grad_norm": 0.17728430032730103,
+      "learning_rate": 9.972688047930772e-05,
+      "loss": 1.07080078125,
+      "step": 17
+    },
+    {
+      "epoch": 0.3050847457627119,
+      "grad_norm": 0.14060959219932556,
+      "learning_rate": 9.965442608816703e-05,
+      "loss": 1.03125,
+      "step": 18
+    },
+    {
+      "epoch": 0.3220338983050847,
+      "grad_norm": 0.13452130556106567,
+      "learning_rate": 9.957349380868764e-05,
+      "loss": 1.116455078125,
+      "step": 19
+    },
+    {
+      "epoch": 0.3389830508474576,
+      "grad_norm": 0.10745736956596375,
+      "learning_rate": 9.948409900630787e-05,
+      "loss": 0.88720703125,
+      "step": 20
+    },
+    {
+      "epoch": 0.3559322033898305,
+      "grad_norm": 0.10258574783802032,
+      "learning_rate": 9.938625865312251e-05,
+      "loss": 0.972412109375,
+      "step": 21
+    },
+    {
+      "epoch": 0.3728813559322034,
+      "grad_norm": 0.10475312918424606,
+      "learning_rate": 9.92799913246606e-05,
+      "loss": 1.1953125,
+      "step": 22
+    },
+    {
+      "epoch": 0.3898305084745763,
+      "grad_norm": 0.09753881394863129,
+      "learning_rate": 9.916531719635881e-05,
+      "loss": 0.908203125,
+      "step": 23
+    },
+    {
+      "epoch": 0.4067796610169492,
+      "grad_norm": 0.11622539907693863,
+      "learning_rate": 9.904225803973094e-05,
+      "loss": 1.06689453125,
+      "step": 24
+    },
+    {
+      "epoch": 0.423728813559322,
+      "grad_norm": 0.10409683734178543,
+      "learning_rate": 9.891083721823461e-05,
+      "loss": 1.007080078125,
+      "step": 25
+    },
+    {
+      "epoch": 0.4406779661016949,
+      "grad_norm": 0.10611087828874588,
+      "learning_rate": 9.877107968283538e-05,
+      "loss": 1.08642578125,
+      "step": 26
+    },
+    {
+      "epoch": 0.4576271186440678,
+      "grad_norm": 0.09127997606992722,
+      "learning_rate": 9.862301196726987e-05,
+      "loss": 1.0029296875,
+      "step": 27
+    },
+    {
+      "epoch": 0.4745762711864407,
+      "grad_norm": 0.08576557040214539,
+      "learning_rate": 9.846666218300807e-05,
+      "loss": 1.02734375,
+      "step": 28
+    },
+    {
+      "epoch": 0.4915254237288136,
+      "grad_norm": 0.0856955274939537,
+      "learning_rate": 9.830206001391626e-05,
+      "loss": 1.03125,
+      "step": 29
+    },
+    {
+      "epoch": 0.5084745762711864,
+      "grad_norm": 0.08397874236106873,
+      "learning_rate": 9.812923671062138e-05,
+      "loss": 1.099365234375,
+      "step": 30
+    },
+    {
+      "epoch": 0.5254237288135594,
+      "grad_norm": 0.08784671872854233,
+      "learning_rate": 9.794822508457784e-05,
+      "loss": 1.072998046875,
+      "step": 31
+    },
+    {
+      "epoch": 0.5423728813559322,
+      "grad_norm": 0.08265256881713867,
+      "learning_rate": 9.775905950183821e-05,
+      "loss": 0.9990234375,
+      "step": 32
+    },
+    {
+      "epoch": 0.559322033898305,
+      "grad_norm": 0.07514134794473648,
+      "learning_rate": 9.756177587652856e-05,
+      "loss": 1.05615234375,
+      "step": 33
+    },
+    {
+      "epoch": 0.576271186440678,
+      "grad_norm": 0.08056832104921341,
+      "learning_rate": 9.735641166402998e-05,
+      "loss": 1.0478515625,
+      "step": 34
+    },
+    {
+      "epoch": 0.5932203389830508,
+      "grad_norm": 0.08656042814254761,
+      "learning_rate": 9.714300585386747e-05,
+      "loss": 0.9912109375,
+      "step": 35
+    },
+    {
+      "epoch": 0.6101694915254238,
+      "grad_norm": 0.0823112279176712,
+      "learning_rate": 9.692159896230756e-05,
+      "loss": 0.940673828125,
+      "step": 36
+    },
+    {
+      "epoch": 0.6271186440677966,
+      "grad_norm": 0.07311712950468063,
+      "learning_rate": 9.669223302466608e-05,
+      "loss": 1.04296875,
+      "step": 37
+    },
+    {
+      "epoch": 0.6440677966101694,
+      "grad_norm": 0.07049518823623657,
+      "learning_rate": 9.645495158732755e-05,
+      "loss": 0.99462890625,
+      "step": 38
+    },
+    {
+      "epoch": 0.6610169491525424,
+      "grad_norm": 0.07307340204715729,
+      "learning_rate": 9.620979969947759e-05,
+      "loss": 0.9482421875,
+      "step": 39
+    },
+    {
+      "epoch": 0.6779661016949152,
+      "grad_norm": 0.07870783656835556,
+      "learning_rate": 9.595682390455015e-05,
+      "loss": 0.82757568359375,
+      "step": 40
+    },
+    {
+      "epoch": 0.6949152542372882,
+      "grad_norm": 0.07178431749343872,
+      "learning_rate": 9.5696072231391e-05,
+      "loss": 0.96826171875,
+      "step": 41
+    },
+    {
+      "epoch": 0.711864406779661,
+      "grad_norm": 0.06541870534420013,
+      "learning_rate": 9.542759418513906e-05,
+      "loss": 0.76641845703125,
+      "step": 42
+    },
+    {
+      "epoch": 0.7288135593220338,
+      "grad_norm": 0.0818762481212616,
+      "learning_rate": 9.515144073782774e-05,
+      "loss": 0.962890625,
+      "step": 43
+    },
+    {
+      "epoch": 0.7457627118644068,
+      "grad_norm": 0.07723236829042435,
+      "learning_rate": 9.486766431870752e-05,
+      "loss": 1.051025390625,
+      "step": 44
+    },
+    {
+      "epoch": 0.7627118644067796,
+      "grad_norm": 0.07016034424304962,
+      "learning_rate": 9.4576318804292e-05,
+      "loss": 0.941162109375,
+      "step": 45
+    },
+    {
+      "epoch": 0.7796610169491526,
+      "grad_norm": 0.07902839779853821,
+      "learning_rate": 9.427745950812917e-05,
+      "loss": 0.94921875,
+      "step": 46
+    },
+    {
+      "epoch": 0.7966101694915254,
+      "grad_norm": 0.07988400012254715,
+      "learning_rate": 9.397114317029975e-05,
+      "loss": 1.051025390625,
+      "step": 47
+    },
+    {
+      "epoch": 0.8135593220338984,
+      "grad_norm": 0.08166942745447159,
+      "learning_rate": 9.365742794664484e-05,
+      "loss": 1.043701171875,
+      "step": 48
+    },
+    {
+      "epoch": 0.8135593220338984,
+      "eval_loss": 0.93798828125,
+      "eval_runtime": 5.5259,
+      "eval_samples_per_second": 0.724,
+      "eval_steps_per_second": 0.181,
+      "step": 48
+    },
+    {
+      "epoch": 0.8305084745762712,
+      "grad_norm": 0.07792344689369202,
+      "learning_rate": 9.333637339772472e-05,
+      "loss": 0.91748046875,
+      "step": 49
+    },
+    {
+      "epoch": 0.847457627118644,
+      "grad_norm": 0.08160696178674698,
+      "learning_rate": 9.300804047751092e-05,
+      "loss": 1.11083984375,
+      "step": 50
+    },
+    {
+      "epoch": 0.864406779661017,
+      "grad_norm": 0.07113709300756454,
+      "learning_rate": 9.267249152181379e-05,
+      "loss": 0.971435546875,
+      "step": 51
+    },
+    {
+      "epoch": 0.8813559322033898,
+      "grad_norm": 0.0816531628370285,
+      "learning_rate": 9.232979023644767e-05,
+      "loss": 0.98974609375,
+      "step": 52
+    },
+    {
+      "epoch": 0.8983050847457628,
+      "grad_norm": 0.08208615332841873,
+      "learning_rate": 9.198000168513604e-05,
+      "loss": 1.03857421875,
+      "step": 53
+    },
+    {
+      "epoch": 0.9152542372881356,
+      "grad_norm": 0.08193928748369217,
+      "learning_rate": 9.162319227715878e-05,
+      "loss": 1.110107421875,
+      "step": 54
+    },
+    {
+      "epoch": 0.9322033898305084,
+      "grad_norm": 0.08443181961774826,
+      "learning_rate": 9.125942975474403e-05,
+      "loss": 1.0693359375,
+      "step": 55
+    },
+    {
+      "epoch": 0.9491525423728814,
+      "grad_norm": 0.08366511017084122,
+      "learning_rate": 9.08887831802069e-05,
+      "loss": 1.046875,
+      "step": 56
+    },
+    {
+      "epoch": 0.9661016949152542,
+      "grad_norm": 0.09129314869642258,
+      "learning_rate": 9.051132292283771e-05,
+      "loss": 0.986572265625,
+      "step": 57
+    },
+    {
+      "epoch": 0.9830508474576272,
+      "grad_norm": 0.08283063024282455,
+      "learning_rate": 9.01271206455419e-05,
+      "loss": 0.98095703125,
+      "step": 58
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.07913633435964584,
+      "learning_rate": 8.973624929123445e-05,
+      "loss": 0.962158203125,
+      "step": 59
+    },
+    {
+      "epoch": 1.0169491525423728,
+      "grad_norm": 0.08337118476629257,
+      "learning_rate": 8.93387830689913e-05,
+      "loss": 0.85302734375,
+      "step": 60
+    },
+    {
+      "epoch": 1.0338983050847457,
+      "grad_norm": 0.09908353537321091,
+      "learning_rate": 8.893479743996034e-05,
+      "loss": 1.0205078125,
+      "step": 61
+    },
+    {
+      "epoch": 1.0508474576271187,
+      "grad_norm": 0.07855907827615738,
+      "learning_rate": 8.852436910303467e-05,
+      "loss": 0.90673828125,
+      "step": 62
+    },
+    {
+      "epoch": 1.0677966101694916,
+      "grad_norm": 1.063303828239441,
+      "learning_rate": 8.810757598029093e-05,
+      "loss": 0.8740234375,
+      "step": 63
+    },
+    {
+      "epoch": 1.0847457627118644,
+      "grad_norm": 0.09119535982608795,
+      "learning_rate": 8.768449720219533e-05,
+      "loss": 0.9609375,
+      "step": 64
+    },
+    {
+      "epoch": 1.1016949152542372,
+      "grad_norm": 0.07633139193058014,
+      "learning_rate": 8.725521309258031e-05,
+      "loss": 1.016357421875,
+      "step": 65
+    },
+    {
+      "epoch": 1.11864406779661,
+      "grad_norm": 0.0819622054696083,
+      "learning_rate": 8.681980515339464e-05,
+      "loss": 1.029296875,
+      "step": 66
+    },
+    {
+      "epoch": 1.1355932203389831,
+      "grad_norm": 0.08978190273046494,
+      "learning_rate": 8.637835604922979e-05,
+      "loss": 0.9619140625,
+      "step": 67
+    },
+    {
+      "epoch": 1.152542372881356,
+      "grad_norm": 0.08815392851829529,
+      "learning_rate": 8.593094959162564e-05,
+      "loss": 0.8486328125,
+      "step": 68
+    },
+    {
+      "epoch": 1.1694915254237288,
+      "grad_norm": 0.08441230654716492,
+      "learning_rate": 8.547767072315835e-05,
+      "loss": 1.02978515625,
+      "step": 69
+    },
+    {
+      "epoch": 1.1864406779661016,
+      "grad_norm": 0.08631148934364319,
+      "learning_rate": 8.50186055013136e-05,
+      "loss": 0.93115234375,
+      "step": 70
+    },
+    {
+      "epoch": 1.2033898305084745,
+      "grad_norm": 0.09572659432888031,
+      "learning_rate": 8.455384108214805e-05,
+      "loss": 0.70166015625,
+      "step": 71
+    },
+    {
+      "epoch": 1.2203389830508475,
+      "grad_norm": 0.0865466445684433,
+      "learning_rate": 8.408346570374233e-05,
+      "loss": 0.91064453125,
+      "step": 72
+    },
+    {
+      "epoch": 1.2372881355932204,
+      "grad_norm": 0.09005347639322281,
+      "learning_rate": 8.360756866944858e-05,
+      "loss": 1.0126953125,
+      "step": 73
+    },
+    {
+      "epoch": 1.2542372881355932,
+      "grad_norm": 0.08672618865966797,
+      "learning_rate": 8.312624033093555e-05,
+      "loss": 0.7626953125,
+      "step": 74
+    },
+    {
+      "epoch": 1.271186440677966,
+      "grad_norm": 0.08496890217065811,
+      "learning_rate": 8.263957207103507e-05,
+      "loss": 1.0,
+      "step": 75
+    },
+    {
+      "epoch": 1.288135593220339,
+      "grad_norm": 0.09614504873752594,
+      "learning_rate": 8.214765628639235e-05,
+      "loss": 0.947265625,
+      "step": 76
+    },
+    {
+      "epoch": 1.305084745762712,
+      "grad_norm": 0.07839664816856384,
+      "learning_rate": 8.165058636992411e-05,
+      "loss": 0.915283203125,
+      "step": 77
+    },
+    {
+      "epoch": 1.3220338983050848,
+      "grad_norm": 0.08440063148736954,
+      "learning_rate": 8.114845669308723e-05,
+      "loss": 0.99365234375,
+      "step": 78
+    },
+    {
+      "epoch": 1.3389830508474576,
+      "grad_norm": 0.08282492309808731,
+      "learning_rate": 8.064136258796198e-05,
+      "loss": 0.7822265625,
+      "step": 79
+    },
+    {
+      "epoch": 1.3559322033898304,
+      "grad_norm": 0.08753400295972824,
+      "learning_rate": 8.012940032915263e-05,
+      "loss": 0.8583984375,
+      "step": 80
+    },
+    {
+      "epoch": 1.3728813559322033,
+      "grad_norm": 0.09745857864618301,
+      "learning_rate": 7.961266711550922e-05,
+      "loss": 1.078125,
+      "step": 81
+    },
+    {
+      "epoch": 1.3898305084745763,
+      "grad_norm": 0.09322620183229446,
+      "learning_rate": 7.909126105167373e-05,
+      "loss": 0.800537109375,
+      "step": 82
+    },
+    {
+      "epoch": 1.4067796610169492,
+      "grad_norm": 0.09690751135349274,
+      "learning_rate": 7.856528112945452e-05,
+      "loss": 0.96142578125,
+      "step": 83
+    },
+    {
+      "epoch": 1.423728813559322,
+      "grad_norm": 0.07993591576814651,
+      "learning_rate": 7.803482720903205e-05,
+      "loss": 0.909912109375,
+      "step": 84
+    },
+    {
+      "epoch": 1.4406779661016949,
+      "grad_norm": 0.08795332163572311,
+      "learning_rate": 7.75e-05,
+      "loss": 0.98095703125,
+      "step": 85
+    },
+    {
+      "epoch": 1.457627118644068,
+      "grad_norm": 0.0931357815861702,
+      "learning_rate": 7.696090104224492e-05,
+      "loss": 0.8984375,
+      "step": 86
+    },
+    {
+      "epoch": 1.4745762711864407,
+      "grad_norm": 0.09056106209754944,
+      "learning_rate": 7.641763268666831e-05,
+      "loss": 0.927001953125,
+      "step": 87
+    },
+    {
+      "epoch": 1.4915254237288136,
+      "grad_norm": 0.10342171043157578,
+      "learning_rate": 7.587029807575482e-05,
+      "loss": 0.9306640625,
+      "step": 88
+    },
+    {
+      "epoch": 1.5084745762711864,
+      "grad_norm": 0.08985871076583862,
+      "learning_rate": 7.531900112399004e-05,
+      "loss": 1.006591796875,
+      "step": 89
+    },
+    {
+      "epoch": 1.5254237288135593,
+      "grad_norm": 0.1016957238316536,
+      "learning_rate": 7.476384649813167e-05,
+      "loss": 0.9736328125,
+      "step": 90
+    },
+    {
+      "epoch": 1.542372881355932,
+      "grad_norm": 0.1086297556757927,
+      "learning_rate": 7.420493959733816e-05,
+      "loss": 0.9091796875,
+      "step": 91
+    },
+    {
+      "epoch": 1.559322033898305,
+      "grad_norm": 0.09838839620351791,
+      "learning_rate": 7.364238653315795e-05,
+      "loss": 0.967041015625,
+      "step": 92
+    },
+    {
+      "epoch": 1.576271186440678,
+      "grad_norm": 0.10762996226549149,
+      "learning_rate": 7.307629410938363e-05,
+      "loss": 0.95849609375,
+      "step": 93
+    },
+    {
+      "epoch": 1.5932203389830508,
+      "grad_norm": 0.09758035838603973,
+      "learning_rate": 7.250676980177468e-05,
+      "loss": 0.89599609375,
+      "step": 94
+    },
+    {
+      "epoch": 1.6101694915254239,
+      "grad_norm": 0.0979393795132637,
+      "learning_rate": 7.193392173765261e-05,
+      "loss": 0.855224609375,
+      "step": 95
+    },
+    {
+      "epoch": 1.6101694915254239,
+      "eval_loss": 0.89306640625,
+      "eval_runtime": 5.5076,
+      "eval_samples_per_second": 0.726,
+      "eval_steps_per_second": 0.182,
+      "step": 95
+    },
+    {
+      "epoch": 1.6271186440677967,
+      "grad_norm": 0.0959092304110527,
+      "learning_rate": 7.135785867537234e-05,
+      "loss": 0.95751953125,
+      "step": 96
+    },
+    {
+      "epoch": 1.6440677966101696,
+      "grad_norm": 0.09253595769405365,
+      "learning_rate": 7.077868998367395e-05,
+      "loss": 0.91943359375,
+      "step": 97
+    },
+    {
+      "epoch": 1.6610169491525424,
+      "grad_norm": 0.09634922444820404,
+      "learning_rate": 7.019652562091827e-05,
+      "loss": 0.871337890625,
+      "step": 98
+    },
+    {
+      "epoch": 1.6779661016949152,
+      "grad_norm": 0.3277416229248047,
+      "learning_rate": 6.961147611421075e-05,
+      "loss": 0.76153564453125,
+      "step": 99
+    },
+    {
+      "epoch": 1.694915254237288,
+      "grad_norm": 0.10589414834976196,
+      "learning_rate": 6.902365253841736e-05,
+      "loss": 0.88671875,
+      "step": 100
+    },
+    {
+      "epoch": 1.711864406779661,
+      "grad_norm": 0.08682512491941452,
+      "learning_rate": 6.843316649507626e-05,
+      "loss": 0.70526123046875,
+      "step": 101
+    },
+    {
+      "epoch": 1.7288135593220337,
+      "grad_norm": 0.1030946597456932,
+      "learning_rate": 6.784013009120974e-05,
+      "loss": 0.8876953125,
+      "step": 102
+    },
+    {
+      "epoch": 1.7457627118644068,
+      "grad_norm": 0.09692194312810898,
+      "learning_rate": 6.724465591804008e-05,
+      "loss": 0.976318359375,
+      "step": 103
+    },
+    {
+      "epoch": 1.7627118644067796,
+      "grad_norm": 0.09669726341962814,
+      "learning_rate": 6.664685702961344e-05,
+      "loss": 0.868408203125,
+      "step": 104
+    },
+    {
+      "epoch": 1.7796610169491527,
+      "grad_norm": 0.10304367542266846,
+      "learning_rate": 6.604684692133597e-05,
+      "loss": 0.87451171875,
+      "step": 105
+    },
+    {
+      "epoch": 1.7966101694915255,
+      "grad_norm": 0.1047707349061966,
+      "learning_rate": 6.544473950842606e-05,
+      "loss": 0.9775390625,
+      "step": 106
+    },
+    {
+      "epoch": 1.8135593220338984,
+      "grad_norm": 0.13497479259967804,
+      "learning_rate": 6.484064910428692e-05,
+      "loss": 0.97412109375,
+      "step": 107
+    },
+    {
+      "epoch": 1.8305084745762712,
+      "grad_norm": 0.1045294776558876,
+      "learning_rate": 6.423469039880354e-05,
+      "loss": 0.849365234375,
+      "step": 108
+    },
+    {
+      "epoch": 1.847457627118644,
+      "grad_norm": 0.10763411968946457,
+      "learning_rate": 6.362697843656823e-05,
+      "loss": 1.0341796875,
+      "step": 109
+    },
+    {
+      "epoch": 1.8644067796610169,
+      "grad_norm": 0.09887196868658066,
+      "learning_rate": 6.301762859503869e-05,
+      "loss": 0.906494140625,
+      "step": 110
+    },
+    {
+      "epoch": 1.8813559322033897,
+      "grad_norm": 0.09416954219341278,
+      "learning_rate": 6.240675656263303e-05,
+      "loss": 0.92138671875,
+      "step": 111
+    },
+    {
+      "epoch": 1.8983050847457628,
+      "grad_norm": 0.10028339922428131,
+      "learning_rate": 6.179447831676566e-05,
+      "loss": 0.96728515625,
+      "step": 112
+    },
+    {
+      "epoch": 1.9152542372881356,
+      "grad_norm": 0.11095006763935089,
+      "learning_rate": 6.118091010182837e-05,
+      "loss": 1.040771484375,
+      "step": 113
+    },
+    {
+      "epoch": 1.9322033898305084,
+      "grad_norm": 0.09702226519584656,
+      "learning_rate": 6.056616840712065e-05,
+      "loss": 0.99658203125,
+      "step": 114
+    },
+    {
+      "epoch": 1.9491525423728815,
+      "grad_norm": 0.11389920860528946,
+      "learning_rate": 5.995036994473357e-05,
+      "loss": 0.984375,
+      "step": 115
+    },
+    {
+      "epoch": 1.9661016949152543,
+      "grad_norm": 0.1546120047569275,
+      "learning_rate": 5.9333631627391384e-05,
+      "loss": 0.921630859375,
+      "step": 116
+    },
+    {
+      "epoch": 1.9830508474576272,
+      "grad_norm": 0.10404060035943985,
+      "learning_rate": 5.8716070546254966e-05,
+      "loss": 0.91796875,
+      "step": 117
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.10162726789712906,
+      "learning_rate": 5.80978039486914e-05,
+      "loss": 0.891357421875,
+      "step": 118
+    },
+    {
+      "epoch": 2.016949152542373,
+      "grad_norm": 0.1013130396604538,
+      "learning_rate": 5.747894921601396e-05,
+      "loss": 0.792236328125,
+      "step": 119
+    },
+    {
+      "epoch": 2.0338983050847457,
+      "grad_norm": 0.12374264746904373,
+      "learning_rate": 5.6859623841196595e-05,
+      "loss": 0.963134765625,
+      "step": 120
+    },
+    {
+      "epoch": 2.0508474576271185,
+      "grad_norm": 0.10125789046287537,
+      "learning_rate": 5.62399454065673e-05,
+      "loss": 0.849853515625,
+      "step": 121
+    },
+    {
+      "epoch": 2.0677966101694913,
+      "grad_norm": 0.10426539182662964,
+      "learning_rate": 5.562003156148434e-05,
+      "loss": 0.819091796875,
+      "step": 122
+    },
+    {
+      "epoch": 2.084745762711864,
+      "grad_norm": 0.11077326536178589,
+      "learning_rate": 5.500000000000001e-05,
+      "loss": 0.9033203125,
+      "step": 123
+    },
+    {
+      "epoch": 2.1016949152542375,
+      "grad_norm": 0.09078076481819153,
+      "learning_rate": 5.437996843851567e-05,
+      "loss": 0.95556640625,
+      "step": 124
+    },
+    {
+      "epoch": 2.1186440677966103,
+      "grad_norm": 0.09942633658647537,
+      "learning_rate": 5.376005459343272e-05,
+      "loss": 0.97265625,
+      "step": 125
+    },
+    {
+      "epoch": 2.135593220338983,
+      "grad_norm": 0.12037667632102966,
+      "learning_rate": 5.314037615880341e-05,
+      "loss": 0.9033203125,
+      "step": 126
+    },
+    {
+      "epoch": 2.152542372881356,
+      "grad_norm": 1.0118740797042847,
+      "learning_rate": 5.2521050783986046e-05,
+      "loss": 0.796142578125,
+      "step": 127
+    },
+    {
+      "epoch": 2.169491525423729,
+      "grad_norm": 0.09780632704496384,
+      "learning_rate": 5.190219605130863e-05,
+      "loss": 0.9775390625,
+      "step": 128
+    },
+    {
+      "epoch": 2.1864406779661016,
+      "grad_norm": 0.1027628555893898,
+      "learning_rate": 5.128392945374505e-05,
+      "loss": 0.875244140625,
+      "step": 129
+    },
+    {
+      "epoch": 2.2033898305084745,
+      "grad_norm": 0.11545059084892273,
+      "learning_rate": 5.066636837260863e-05,
+      "loss": 0.656005859375,
+      "step": 130
+    },
+    {
+      "epoch": 2.2203389830508473,
+      "grad_norm": 0.10182905197143555,
+      "learning_rate": 5.004963005526644e-05,
+      "loss": 0.860107421875,
+      "step": 131
+    },
+    {
+      "epoch": 2.23728813559322,
+      "grad_norm": 0.337242066860199,
+      "learning_rate": 4.9433831592879355e-05,
+      "loss": 0.95751953125,
+      "step": 132
+    },
+    {
+      "epoch": 2.2542372881355934,
+      "grad_norm": 0.12530653178691864,
+      "learning_rate": 4.881908989817163e-05,
+      "loss": 0.71533203125,
+      "step": 133
+    },
+    {
+      "epoch": 2.2711864406779663,
+      "grad_norm": 0.1110646054148674,
+      "learning_rate": 4.8205521683234335e-05,
+      "loss": 0.947265625,
+      "step": 134
+    },
+    {
+      "epoch": 2.288135593220339,
+      "grad_norm": 0.09985275566577911,
+      "learning_rate": 4.7593243437366975e-05,
+      "loss": 0.897216796875,
+      "step": 135
+    },
+    {
+      "epoch": 2.305084745762712,
+      "grad_norm": 0.09477593004703522,
+      "learning_rate": 4.698237140496132e-05,
+      "loss": 0.86962890625,
+      "step": 136
+    },
+    {
+      "epoch": 2.3220338983050848,
+      "grad_norm": 0.10785605013370514,
+      "learning_rate": 4.6373021563431784e-05,
+      "loss": 0.947998046875,
+      "step": 137
+    },
+    {
+      "epoch": 2.3389830508474576,
+      "grad_norm": 0.1287638396024704,
+      "learning_rate": 4.576530960119646e-05,
+      "loss": 0.7392578125,
+      "step": 138
+    },
+    {
+      "epoch": 2.3559322033898304,
+      "grad_norm": 0.09915657341480255,
+      "learning_rate": 4.515935089571309e-05,
+      "loss": 0.81396484375,
+      "step": 139
+    },
+    {
+      "epoch": 2.3728813559322033,
+      "grad_norm": 0.11962717771530151,
+      "learning_rate": 4.455526049157396e-05,
+      "loss": 1.0302734375,
+      "step": 140
+    },
+    {
+      "epoch": 2.389830508474576,
+      "grad_norm": 0.1173805221915245,
+      "learning_rate": 4.395315307866405e-05,
+      "loss": 0.757080078125,
+      "step": 141
+    },
+    {
+      "epoch": 2.406779661016949,
+      "grad_norm": 0.10178910940885544,
+      "learning_rate": 4.3353142970386564e-05,
+      "loss": 0.91796875,
+      "step": 142
+    },
+    {
+      "epoch": 2.406779661016949,
+      "eval_loss": 0.876953125,
+      "eval_runtime": 5.5331,
+      "eval_samples_per_second": 0.723,
+      "eval_steps_per_second": 0.181,
+      "step": 142
+    },
+    {
+      "epoch": 2.423728813559322,
+      "grad_norm": 0.0964282676577568,
+      "learning_rate": 4.275534408195991e-05,
+      "loss": 0.869140625,
+      "step": 143
+    },
+    {
+      "epoch": 2.440677966101695,
+      "grad_norm": 0.09992555528879166,
+      "learning_rate": 4.215986990879027e-05,
+      "loss": 0.93701171875,
+      "step": 144
+    },
+    {
+      "epoch": 2.457627118644068,
+      "grad_norm": 0.11470576375722885,
+      "learning_rate": 4.156683350492376e-05,
+      "loss": 0.85498046875,
+      "step": 145
+    },
+    {
+      "epoch": 2.4745762711864407,
+      "grad_norm": 0.10724995285272598,
+      "learning_rate": 4.0976347461582654e-05,
+      "loss": 0.8857421875,
+      "step": 146
+    },
+    {
+      "epoch": 2.4915254237288136,
+      "grad_norm": 0.10291863232851028,
+      "learning_rate": 4.0388523885789256e-05,
+      "loss": 0.88671875,
+      "step": 147
+    },
+    {
+      "epoch": 2.5084745762711864,
+      "grad_norm": 0.12115908414125443,
+      "learning_rate": 3.980347437908175e-05,
+      "loss": 0.9638671875,
+      "step": 148
+    },
+    {
+      "epoch": 2.5254237288135593,
+      "grad_norm": 0.12119631469249725,
+      "learning_rate": 3.922131001632606e-05,
+      "loss": 0.93115234375,
+      "step": 149
+    },
+    {
+      "epoch": 2.542372881355932,
+      "grad_norm": 0.11702482402324677,
+      "learning_rate": 3.864214132462765e-05,
+      "loss": 0.866943359375,
+      "step": 150
+    },
+    {
+      "epoch": 2.559322033898305,
+      "grad_norm": 0.10389980673789978,
+      "learning_rate": 3.8066078262347406e-05,
+      "loss": 0.92919921875,
+      "step": 151
+    },
+    {
+      "epoch": 2.576271186440678,
+      "grad_norm": 0.1386137753725052,
+      "learning_rate": 3.749323019822534e-05,
+      "loss": 0.9169921875,
+      "step": 152
+    },
+    {
+      "epoch": 2.593220338983051,
+      "grad_norm": 0.10683158785104752,
+      "learning_rate": 3.692370589061639e-05,
+      "loss": 0.85595703125,
+      "step": 153
+    },
+    {
+      "epoch": 2.610169491525424,
+      "grad_norm": 0.11181148141622543,
+      "learning_rate": 3.6357613466842054e-05,
+      "loss": 0.815673828125,
+      "step": 154
+    },
+    {
+      "epoch": 2.6271186440677967,
+      "grad_norm": 0.12605640292167664,
+      "learning_rate": 3.579506040266184e-05,
+      "loss": 0.919677734375,
+      "step": 155
+    },
+    {
+      "epoch": 2.6440677966101696,
+      "grad_norm": 0.11007009446620941,
+      "learning_rate": 3.523615350186834e-05,
+      "loss": 0.88134765625,
+      "step": 156
+    },
+    {
+      "epoch": 2.6610169491525424,
+      "grad_norm": 0.0949372947216034,
+      "learning_rate": 3.468099887600999e-05,
+      "loss": 0.833984375,
+      "step": 157
+    },
+    {
+      "epoch": 2.6779661016949152,
+      "grad_norm": 0.09774491935968399,
+      "learning_rate": 3.412970192424517e-05,
+      "loss": 0.73193359375,
+      "step": 158
+    },
+    {
+      "epoch": 2.694915254237288,
+      "grad_norm": 0.1312067061662674,
+      "learning_rate": 3.358236731333169e-05,
+      "loss": 0.84912109375,
+      "step": 159
+    },
+    {
+      "epoch": 2.711864406779661,
+      "grad_norm": 0.09475670754909515,
+      "learning_rate": 3.3039098957755097e-05,
+      "loss": 0.67608642578125,
+      "step": 160
+    },
+    {
+      "epoch": 2.7288135593220337,
+      "grad_norm": 0.11485911160707474,
+      "learning_rate": 3.250000000000001e-05,
+      "loss": 0.848388671875,
+      "step": 161
+    },
+    {
+      "epoch": 2.7457627118644066,
+      "grad_norm": 0.10541412979364395,
+      "learning_rate": 3.1965172790967965e-05,
+      "loss": 0.9404296875,
+      "step": 162
+    },
+    {
+      "epoch": 2.7627118644067794,
+      "grad_norm": 0.2602388262748718,
+      "learning_rate": 3.14347188705455e-05,
+      "loss": 0.83349609375,
+      "step": 163
+    },
+    {
+      "epoch": 2.7796610169491527,
+      "grad_norm": 0.10894317924976349,
+      "learning_rate": 3.0908738948326285e-05,
+      "loss": 0.838134765625,
+      "step": 164
+    },
+    {
+      "epoch": 2.7966101694915255,
+      "grad_norm": 0.11598169803619385,
+      "learning_rate": 3.0387332884490805e-05,
+      "loss": 0.94140625,
+      "step": 165
+    },
+    {
+      "epoch": 2.8135593220338984,
+      "grad_norm": 0.09514643251895905,
+      "learning_rate": 2.9870599670847366e-05,
+      "loss": 0.940673828125,
+      "step": 166
+    },
+    {
+      "epoch": 2.830508474576271,
+      "grad_norm": 0.11295430362224579,
+      "learning_rate": 2.9358637412038027e-05,
+      "loss": 0.8154296875,
+      "step": 167
+    },
+    {
+      "epoch": 2.847457627118644,
+      "grad_norm": 0.13384811580181122,
+      "learning_rate": 2.885154330691278e-05,
+      "loss": 0.9990234375,
+      "step": 168
+    },
+    {
+      "epoch": 2.864406779661017,
+      "grad_norm": 0.10941619426012039,
+      "learning_rate": 2.8349413630075906e-05,
+      "loss": 0.87255859375,
+      "step": 169
+    },
+    {
+      "epoch": 2.8813559322033897,
+      "grad_norm": 0.12930384278297424,
+      "learning_rate": 2.785234371360766e-05,
+      "loss": 0.88671875,
+      "step": 170
+    },
+    {
+      "epoch": 2.898305084745763,
+      "grad_norm": 0.11116838455200195,
+      "learning_rate": 2.736042792896495e-05,
+      "loss": 0.932373046875,
+      "step": 171
+    },
+    {
+      "epoch": 2.915254237288136,
+      "grad_norm": 0.12374906986951828,
+      "learning_rate": 2.6873759669064476e-05,
+      "loss": 1.005126953125,
+      "step": 172
+    },
+    {
+      "epoch": 2.9322033898305087,
+      "grad_norm": 0.11184250563383102,
+      "learning_rate": 2.639243133055145e-05,
+      "loss": 0.96240234375,
+      "step": 173
+    },
+    {
+      "epoch": 2.9491525423728815,
+      "grad_norm": 0.10277713090181351,
+      "learning_rate": 2.5916534296257655e-05,
+      "loss": 0.953125,
+      "step": 174
+    },
+    {
+      "epoch": 2.9661016949152543,
+      "grad_norm": 0.11024855822324753,
+      "learning_rate": 2.5446158917851958e-05,
+      "loss": 0.890869140625,
+      "step": 175
+    },
+    {
+      "epoch": 2.983050847457627,
+      "grad_norm": 0.11626192927360535,
+      "learning_rate": 2.498139449868641e-05,
+      "loss": 0.8876953125,
+      "step": 176
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.11005569249391556,
+      "learning_rate": 2.4522329276841663e-05,
+      "loss": 0.8564453125,
+      "step": 177
+    },
+    {
+      "epoch": 3.016949152542373,
+      "grad_norm": 0.10232047736644745,
+      "learning_rate": 2.4069050408374373e-05,
+      "loss": 0.759765625,
+      "step": 178
+    },
+    {
+      "epoch": 3.0338983050847457,
+      "grad_norm": 0.11233022063970566,
+      "learning_rate": 2.362164395077021e-05,
+      "loss": 0.93603515625,
+      "step": 179
+    },
+    {
+      "epoch": 3.0508474576271185,
+      "grad_norm": 0.10281368345022202,
+      "learning_rate": 2.3180194846605367e-05,
+      "loss": 0.82080078125,
+      "step": 180
+    },
+    {
+      "epoch": 3.0677966101694913,
+      "grad_norm": 0.10223117470741272,
+      "learning_rate": 2.2744786907419703e-05,
+      "loss": 0.792236328125,
+      "step": 181
+    },
+    {
+      "epoch": 3.084745762711864,
+      "grad_norm": 0.10896459966897964,
+      "learning_rate": 2.231550279780468e-05,
+      "loss": 0.87158203125,
+      "step": 182
+    },
+    {
+      "epoch": 3.1016949152542375,
+      "grad_norm": 0.10090362280607224,
+      "learning_rate": 2.189242401970908e-05,
+      "loss": 0.928466796875,
+      "step": 183
+    },
+    {
+      "epoch": 3.1186440677966103,
+      "grad_norm": 0.09479139000177383,
+      "learning_rate": 2.1475630896965336e-05,
+      "loss": 0.946533203125,
+      "step": 184
+    },
+    {
+      "epoch": 3.135593220338983,
+      "grad_norm": 0.1425022929906845,
+      "learning_rate": 2.1065202560039677e-05,
+      "loss": 0.875244140625,
+      "step": 185
+    },
+    {
+      "epoch": 3.152542372881356,
+      "grad_norm": 0.10372081398963928,
+      "learning_rate": 2.0661216931008714e-05,
+      "loss": 0.773681640625,
+      "step": 186
+    },
+    {
+      "epoch": 3.169491525423729,
+      "grad_norm": 0.09539317339658737,
+      "learning_rate": 2.026375070876556e-05,
+      "loss": 0.95068359375,
+      "step": 187
+    },
+    {
+      "epoch": 3.1864406779661016,
+      "grad_norm": 0.10878961533308029,
+      "learning_rate": 1.987287935445811e-05,
+      "loss": 0.847900390625,
+      "step": 188
+    },
+    {
+      "epoch": 3.2033898305084745,
+      "grad_norm": 0.11231493949890137,
+      "learning_rate": 1.9488677077162295e-05,
+      "loss": 0.631103515625,
+      "step": 189
+    },
+    {
+      "epoch": 3.2033898305084745,
+      "eval_loss": 0.869140625,
+      "eval_runtime": 5.5277,
+      "eval_samples_per_second": 0.724,
+      "eval_steps_per_second": 0.181,
+      "step": 189
+    },
+    {
+      "epoch": 3.2203389830508473,
+      "grad_norm": 0.10731599479913712,
+      "learning_rate": 1.9111216819793097e-05,
+      "loss": 0.834228515625,
+      "step": 190
+    },
+    {
+      "epoch": 3.23728813559322,
+      "grad_norm": 0.10284540802240372,
+      "learning_rate": 1.8740570245255984e-05,
+      "loss": 0.92919921875,
+      "step": 191
+    },
+    {
+      "epoch": 3.2542372881355934,
+      "grad_norm": 0.11073050647974014,
+      "learning_rate": 1.837680772284123e-05,
+      "loss": 0.689453125,
+      "step": 192
+    },
+    {
+      "epoch": 3.2711864406779663,
+      "grad_norm": 0.12270122021436691,
+      "learning_rate": 1.8019998314863974e-05,
+      "loss": 0.91943359375,
+      "step": 193
+    },
+    {
+      "epoch": 3.288135593220339,
+      "grad_norm": 0.10275924950838089,
+      "learning_rate": 1.767020976355234e-05,
+      "loss": 0.873779296875,
+      "step": 194
+    },
+    {
+      "epoch": 3.305084745762712,
+      "grad_norm": 0.097442127764225,
+      "learning_rate": 1.7327508478186218e-05,
+      "loss": 0.845458984375,
+      "step": 195
+    },
+    {
+      "epoch": 3.3220338983050848,
+      "grad_norm": 0.09888269007205963,
+      "learning_rate": 1.6991959522489082e-05,
+      "loss": 0.921875,
+      "step": 196
+    },
+    {
+      "epoch": 3.3389830508474576,
+      "grad_norm": 0.10165548324584961,
+      "learning_rate": 1.6663626602275288e-05,
+      "loss": 0.71728515625,
+      "step": 197
+    },
+    {
+      "epoch": 3.3559322033898304,
+      "grad_norm": 0.1232258751988411,
+      "learning_rate": 1.6342572053355166e-05,
+      "loss": 0.793212890625,
+      "step": 198
+    },
+    {
+      "epoch": 3.3728813559322033,
+      "grad_norm": 0.10164085775613785,
+      "learning_rate": 1.602885682970026e-05,
+      "loss": 1.0048828125,
+      "step": 199
+    },
+    {
+      "epoch": 3.389830508474576,
+      "grad_norm": 0.10870424658060074,
+      "learning_rate": 1.5722540491870837e-05,
+      "loss": 0.734130859375,
+      "step": 200
+    },
+    {
+      "epoch": 3.406779661016949,
+      "grad_norm": 0.09911137819290161,
+      "learning_rate": 1.5423681195707997e-05,
+      "loss": 0.89501953125,
+      "step": 201
+    },
+    {
+      "epoch": 3.423728813559322,
+      "grad_norm": 0.09735211730003357,
+      "learning_rate": 1.5132335681292492e-05,
+      "loss": 0.849365234375,
+      "step": 202
+    },
+    {
+      "epoch": 3.440677966101695,
+      "grad_norm": 0.11244349926710129,
+      "learning_rate": 1.484855926217227e-05,
+      "loss": 0.91259765625,
+      "step": 203
+    },
+    {
+      "epoch": 3.457627118644068,
+      "grad_norm": 0.10653837025165558,
+      "learning_rate": 1.4572405814860954e-05,
+      "loss": 0.831298828125,
+      "step": 204
+    },
+    {
+      "epoch": 3.4745762711864407,
+      "grad_norm": 0.1004338338971138,
+      "learning_rate": 1.4303927768609015e-05,
+      "loss": 0.86279296875,
+      "step": 205
+    },
+    {
+      "epoch": 3.4915254237288136,
+      "grad_norm": 0.11053711920976639,
+      "learning_rate": 1.4043176095449841e-05,
+      "loss": 0.8623046875,
+      "step": 206
+    },
+    {
+      "epoch": 3.5084745762711864,
+      "grad_norm": 0.11450106650590897,
+      "learning_rate": 1.3790200300522413e-05,
+      "loss": 0.94287109375,
+      "step": 207
+    },
+    {
+      "epoch": 3.5254237288135593,
+      "grad_norm": 0.1193515956401825,
+      "learning_rate": 1.354504841267246e-05,
+      "loss": 0.91064453125,
+      "step": 208
+    },
+    {
+      "epoch": 3.542372881355932,
+      "grad_norm": 0.10841777920722961,
+      "learning_rate": 1.330776697533392e-05,
+      "loss": 0.845947265625,
+      "step": 209
+    },
+    {
+      "epoch": 3.559322033898305,
+      "grad_norm": 0.09431479871273041,
+      "learning_rate": 1.307840103769245e-05,
+      "loss": 0.90966796875,
+      "step": 210
+    },
+    {
+      "epoch": 3.576271186440678,
+      "grad_norm": 0.12500523030757904,
+      "learning_rate": 1.2856994146132542e-05,
+      "loss": 0.89599609375,
+      "step": 211
+    },
+    {
+      "epoch": 3.593220338983051,
+      "grad_norm": 0.10567139089107513,
+      "learning_rate": 1.2643588335970022e-05,
+      "loss": 0.83544921875,
+      "step": 212
+    },
+    {
+      "epoch": 3.610169491525424,
+      "grad_norm": 0.10635709017515182,
+      "learning_rate": 1.2438224123471442e-05,
+      "loss": 0.794921875,
+      "step": 213
+    },
+    {
+      "epoch": 3.6271186440677967,
+      "grad_norm": 0.10316949337720871,
+      "learning_rate": 1.2240940498161797e-05,
+      "loss": 0.900146484375,
+      "step": 214
+    },
+    {
+      "epoch": 3.6440677966101696,
+      "grad_norm": 0.09647554159164429,
+      "learning_rate": 1.2051774915422163e-05,
+      "loss": 0.8642578125,
+      "step": 215
+    },
+    {
+      "epoch": 3.6610169491525424,
+      "grad_norm": 0.09970184415578842,
+      "learning_rate": 1.1870763289378629e-05,
+      "loss": 0.81787109375,
+      "step": 216
+    },
+    {
+      "epoch": 3.6779661016949152,
+      "grad_norm": 0.09053965657949448,
+      "learning_rate": 1.1697939986083733e-05,
+      "loss": 0.71661376953125,
+      "step": 217
+    },
+    {
+      "epoch": 3.694915254237288,
+      "grad_norm": 0.10457823425531387,
+      "learning_rate": 1.1533337816991932e-05,
+      "loss": 0.83056640625,
+      "step": 218
+    },
+    {
+      "epoch": 3.711864406779661,
+      "grad_norm": 0.09273640811443329,
+      "learning_rate": 1.1376988032730134e-05,
+      "loss": 0.6630859375,
+      "step": 219
+    },
+    {
+      "epoch": 3.7288135593220337,
+      "grad_norm": 0.10779716819524765,
+      "learning_rate": 1.1228920317164623e-05,
+      "loss": 0.833251953125,
+      "step": 220
+    },
+    {
+      "epoch": 3.7457627118644066,
+      "grad_norm": 0.11510983109474182,
+      "learning_rate": 1.1089162781765398e-05,
+      "loss": 0.923095703125,
+      "step": 221
+    },
+    {
+      "epoch": 3.7627118644067794,
+      "grad_norm": 0.10284290462732315,
+      "learning_rate": 1.0957741960269049e-05,
+      "loss": 0.81591796875,
+      "step": 222
+    },
+    {
+      "epoch": 3.7796610169491527,
+      "grad_norm": 0.11504746228456497,
+      "learning_rate": 1.0834682803641197e-05,
+      "loss": 0.820068359375,
+      "step": 223
+    },
+    {
+      "epoch": 3.7966101694915255,
+      "grad_norm": 0.10790020227432251,
+      "learning_rate": 1.0720008675339404e-05,
+      "loss": 0.925048828125,
+      "step": 224
+    },
+    {
+      "epoch": 3.8135593220338984,
+      "grad_norm": 0.10109377652406693,
+      "learning_rate": 1.0613741346877497e-05,
+      "loss": 0.925537109375,
+      "step": 225
+    },
+    {
+      "epoch": 3.830508474576271,
+      "grad_norm": 0.10490377247333527,
+      "learning_rate": 1.0515900993692127e-05,
+      "loss": 0.800537109375,
+      "step": 226
+    },
+    {
+      "epoch": 3.847457627118644,
+      "grad_norm": 0.10508555173873901,
+      "learning_rate": 1.0426506191312355e-05,
+      "loss": 0.98046875,
+      "step": 227
+    },
+    {
+      "epoch": 3.864406779661017,
+      "grad_norm": 0.10123743861913681,
+      "learning_rate": 1.0345573911832976e-05,
+      "loss": 0.85791015625,
+      "step": 228
+    },
+    {
+      "epoch": 3.8813559322033897,
+      "grad_norm": 0.10421520471572876,
+      "learning_rate": 1.0273119520692275e-05,
+      "loss": 0.87060546875,
+      "step": 229
+    },
+    {
+      "epoch": 3.898305084745763,
+      "grad_norm": 0.1495855748653412,
+      "learning_rate": 1.020915677375483e-05,
+      "loss": 0.917724609375,
+      "step": 230
+    },
+    {
+      "epoch": 3.915254237288136,
+      "grad_norm": 0.10790368914604187,
+      "learning_rate": 1.0153697814699859e-05,
+      "loss": 0.990966796875,
+      "step": 231
+    },
+    {
+      "epoch": 3.9322033898305087,
+      "grad_norm": 0.10553882271051407,
+      "learning_rate": 1.01067531727157e-05,
+      "loss": 0.94677734375,
+      "step": 232
+    },
+    {
+      "epoch": 3.9491525423728815,
+      "grad_norm": 0.1143532320857048,
+      "learning_rate": 1.0068331760500774e-05,
+      "loss": 0.939208984375,
+      "step": 233
+    },
+    {
+      "epoch": 3.9661016949152543,
+      "grad_norm": 0.10906478762626648,
+      "learning_rate": 1.0038440872571456e-05,
+      "loss": 0.877197265625,
+      "step": 234
+    },
+    {
+      "epoch": 3.983050847457627,
+      "grad_norm": 0.1037352979183197,
+      "learning_rate": 1.0017086183877188e-05,
+      "loss": 0.87548828125,
+      "step": 235
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.10856913775205612,
+      "learning_rate": 1.0004271748723041e-05,
+      "loss": 0.84228515625,
+      "step": 236
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.8642578125,
+      "eval_runtime": 5.5072,
+      "eval_samples_per_second": 0.726,
+      "eval_steps_per_second": 0.182,
+      "step": 236
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 236,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.0521223963556184e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}