Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+g1_blog_post/250M/tokenizer.json filter=lfs diff=lfs merge=lfs -text

g1_blog_post/250M/README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen3-8B-Base
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: 250M
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 250M
+
+This model is a fine-tuned version of [/lus/flare/projects/MultiModalLLM/model-weights/Qwen/Qwen3-8B-Base](https://huggingface.co//lus/flare/projects/MultiModalLLM/model-weights/Qwen/Qwen3-8B-Base) on the rv5_g1_blog_post and the instruct_replay datasets.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 12
+- total_train_batch_size: 12
+- total_eval_batch_size: 96
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: warmup_stable_decay
+- lr_scheduler_warmup_steps: 178
+- training_steps: 2797
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.57.6
+- Pytorch 2.10.0a0+git449b176
+- Datasets 4.5.0
+- Tokenizers 0.22.2

g1_blog_post/250M/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

g1_blog_post/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.25071710290426674,
+    "total_flos": 1.2485883762455871e+19,
+    "train_loss": 0.08493010026879254,
+    "train_runtime": 2880.0845,
+    "train_samples_per_second": 11.654,
+    "train_steps_per_second": 0.971
+}

g1_blog_post/250M/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

g1_blog_post/250M/config.json

@@ -0,0 +1,68 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 12288,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 36,
+  "model_type": "qwen3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

g1_blog_post/250M/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "eos_token_id": [
+    151643
+  ],
+  "max_new_tokens": 2048,
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

g1_blog_post/250M/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:459f70421aba040fca2bc15f41f743142755fec88b387b188c02e86e3c15cdc8
+size 4902257696

g1_blog_post/250M/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea5fb8869a1d12b9733b5d8937d2a1f97742c87d75e5bde1d95c0b7e8c0e792
+size 4915960368

g1_blog_post/250M/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03660f367f63d345b6c49343d557a0a99664fb72529163fc9f4a0d44fd980616
+size 4983068496

g1_blog_post/250M/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14abcd7aaee2ec06b9d3336f27a4ed84042ae9389d5bb2d90c11651d8f3c996
+size 1580230264

g1_blog_post/250M/model.safetensors.index.json

@@ -0,0 +1,407 @@
+{
+  "metadata": {
+    "total_parameters": 8190735360,
+    "total_size": 16381470720
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

g1_blog_post/250M/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_blog_post/250M/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

g1_blog_post/250M/tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

g1_blog_post/250M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.25071710290426674,
+    "total_flos": 1.2485883762455871e+19,
+    "train_loss": 0.08493010026879254,
+    "train_runtime": 2880.0845,
+    "train_samples_per_second": 11.654,
+    "train_steps_per_second": 0.971
+}

g1_blog_post/250M/trainer_log.jsonl

@@ -0,0 +1,26 @@
+{"current_steps": 2550, "total_steps": 2797, "loss": 0.9168, "lr": 1.997261400182901e-05, "epoch": 0.22857655073503047, "percentage": 91.17, "elapsed_time": "0:01:21", "remaining_time": "0:00:07"}
+{"current_steps": 2560, "total_steps": 2797, "loss": 0.9495, "lr": 1.9805800746810423e-05, "epoch": 0.22947292936536393, "percentage": 91.53, "elapsed_time": "0:03:14", "remaining_time": "0:00:18"}
+{"current_steps": 2570, "total_steps": 2797, "loss": 0.9421, "lr": 1.9489934220110274e-05, "epoch": 0.23036930799569738, "percentage": 91.88, "elapsed_time": "0:05:06", "remaining_time": "0:00:27"}
+{"current_steps": 2580, "total_steps": 2797, "loss": 0.9519, "lr": 1.9029840364820392e-05, "epoch": 0.23126568662603084, "percentage": 92.24, "elapsed_time": "0:06:59", "remaining_time": "0:00:35"}
+{"current_steps": 2590, "total_steps": 2797, "loss": 0.9187, "lr": 1.8432548690479908e-05, "epoch": 0.2321620652563643, "percentage": 92.6, "elapsed_time": "0:08:52", "remaining_time": "0:00:42"}
+{"current_steps": 2600, "total_steps": 2797, "loss": 0.9465, "lr": 1.7707184873239772e-05, "epoch": 0.23305844388669775, "percentage": 92.96, "elapsed_time": "0:10:44", "remaining_time": "0:00:48"}
+{"current_steps": 2610, "total_steps": 2797, "loss": 0.9534, "lr": 1.6864831329902565e-05, "epoch": 0.2339548225170312, "percentage": 93.31, "elapsed_time": "0:12:37", "remaining_time": "0:00:54"}
+{"current_steps": 2620, "total_steps": 2797, "loss": 0.9365, "lr": 1.591835789604667e-05, "epoch": 0.23485120114736466, "percentage": 93.67, "elapsed_time": "0:14:29", "remaining_time": "0:00:58"}
+{"current_steps": 2630, "total_steps": 2797, "loss": 0.9476, "lr": 1.4882225195206459e-05, "epoch": 0.2357475797776981, "percentage": 94.03, "elapsed_time": "0:16:22", "remaining_time": "0:01:02"}
+{"current_steps": 2640, "total_steps": 2797, "loss": 0.9418, "lr": 1.3772263703317695e-05, "epoch": 0.23664395840803154, "percentage": 94.39, "elapsed_time": "0:18:14", "remaining_time": "0:01:05"}
+{"current_steps": 2650, "total_steps": 2797, "loss": 0.9498, "lr": 1.2605431883975547e-05, "epoch": 0.237540337038365, "percentage": 94.74, "elapsed_time": "0:20:07", "remaining_time": "0:01:06"}
+{"current_steps": 2660, "total_steps": 2797, "loss": 0.925, "lr": 1.1399557089817574e-05, "epoch": 0.23843671566869845, "percentage": 95.1, "elapsed_time": "0:22:00", "remaining_time": "0:01:07"}
+{"current_steps": 2670, "total_steps": 2797, "loss": 0.9592, "lr": 1.0173063188650463e-05, "epoch": 0.2393330942990319, "percentage": 95.46, "elapsed_time": "0:23:52", "remaining_time": "0:01:08"}
+{"current_steps": 2680, "total_steps": 2797, "loss": 0.9319, "lr": 8.944689075764396e-06, "epoch": 0.24022947292936536, "percentage": 95.82, "elapsed_time": "0:25:45", "remaining_time": "0:01:07"}
+{"current_steps": 2690, "total_steps": 2797, "loss": 0.9464, "lr": 7.73320237312362e-06, "epoch": 0.24112585155969882, "percentage": 96.17, "elapsed_time": "0:27:37", "remaining_time": "0:01:05"}
+{"current_steps": 2700, "total_steps": 2797, "loss": 0.9257, "lr": 6.557112689658629e-06, "epoch": 0.24202223019003227, "percentage": 96.53, "elapsed_time": "0:29:30", "remaining_time": "0:01:03"}
+{"current_steps": 2710, "total_steps": 2797, "loss": 0.9491, "lr": 5.434388823591213e-06, "epoch": 0.24291860882036573, "percentage": 96.89, "elapsed_time": "0:31:22", "remaining_time": "0:01:00"}
+{"current_steps": 2720, "total_steps": 2797, "loss": 0.9357, "lr": 4.382184227495424e-06, "epoch": 0.24381498745069918, "percentage": 97.25, "elapsed_time": "0:33:15", "remaining_time": "0:00:56"}
+{"current_steps": 2730, "total_steps": 2797, "loss": 0.9352, "lr": 3.4165749305560203e-06, "epoch": 0.24471136608103264, "percentage": 97.6, "elapsed_time": "0:35:07", "remaining_time": "0:00:51"}
+{"current_steps": 2740, "total_steps": 2797, "loss": 0.9424, "lr": 2.5523139221595556e-06, "epoch": 0.2456077447113661, "percentage": 97.96, "elapsed_time": "0:37:00", "remaining_time": "0:00:46"}
+{"current_steps": 2750, "total_steps": 2797, "loss": 0.9466, "lr": 1.8026057494499821e-06, "epoch": 0.24650412334169952, "percentage": 98.32, "elapsed_time": "0:38:52", "remaining_time": "0:00:39"}
+{"current_steps": 2760, "total_steps": 2797, "loss": 0.956, "lr": 1.178904772643223e-06, "epoch": 0.24740050197203298, "percentage": 98.68, "elapsed_time": "0:40:45", "remaining_time": "0:00:32"}
+{"current_steps": 2770, "total_steps": 2797, "loss": 0.9211, "lr": 6.90740160441754e-07, "epoch": 0.24829688060236643, "percentage": 99.03, "elapsed_time": "0:42:37", "remaining_time": "0:00:24"}
+{"current_steps": 2780, "total_steps": 2797, "loss": 0.9277, "lr": 3.45570299343632e-07, "epoch": 0.2491932592326999, "percentage": 99.39, "elapsed_time": "0:44:30", "remaining_time": "0:00:16"}
+{"current_steps": 2790, "total_steps": 2797, "loss": 0.9296, "lr": 1.486688412425152e-07, "epoch": 0.25008963786303334, "percentage": 99.75, "elapsed_time": "0:46:23", "remaining_time": "0:00:06"}
+{"current_steps": 2797, "total_steps": 2797, "epoch": 0.25071710290426674, "percentage": 100.0, "elapsed_time": "0:48:00", "remaining_time": "0:00:00"}

g1_blog_post/250M/trainer_state.json

@@ -0,0 +1,1996 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.25071710290426674,
+  "eval_steps": 500,
+  "global_step": 2797,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008963786303334528,
+      "grad_norm": 4.222406387329102,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.3527,
+      "step": 10
+    },
+    {
+      "epoch": 0.0017927572606669057,
+      "grad_norm": 2.6807146072387695,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 1.3352,
+      "step": 20
+    },
+    {
+      "epoch": 0.0026891358910003586,
+      "grad_norm": 1.6670352220535278,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.2309,
+      "step": 30
+    },
+    {
+      "epoch": 0.0035855145213338113,
+      "grad_norm": 1.354493498802185,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.1554,
+      "step": 40
+    },
+    {
+      "epoch": 0.0044818931516672645,
+      "grad_norm": 1.4760373830795288,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 1.1138,
+      "step": 50
+    },
+    {
+      "epoch": 0.005378271782000717,
+      "grad_norm": 1.1128770112991333,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 1.0987,
+      "step": 60
+    },
+    {
+      "epoch": 0.00627465041233417,
+      "grad_norm": 1.0019757747650146,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 1.0886,
+      "step": 70
+    },
+    {
+      "epoch": 0.007171029042667623,
+      "grad_norm": 0.9669538140296936,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 1.0782,
+      "step": 80
+    },
+    {
+      "epoch": 0.008067407673001075,
+      "grad_norm": 0.961510181427002,
+      "learning_rate": 1.005e-05,
+      "loss": 1.0558,
+      "step": 90
+    },
+    {
+      "epoch": 0.008963786303334529,
+      "grad_norm": 1.238696813583374,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 1.0627,
+      "step": 100
+    },
+    {
+      "epoch": 0.00986016493366798,
+      "grad_norm": 0.9661775231361389,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 1.048,
+      "step": 110
+    },
+    {
+      "epoch": 0.010756543564001434,
+      "grad_norm": 1.0016547441482544,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 1.0484,
+      "step": 120
+    },
+    {
+      "epoch": 0.011652922194334886,
+      "grad_norm": 0.9316088557243347,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 1.0444,
+      "step": 130
+    },
+    {
+      "epoch": 0.01254930082466834,
+      "grad_norm": 1.016417384147644,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 1.0556,
+      "step": 140
+    },
+    {
+      "epoch": 0.013445679455001793,
+      "grad_norm": 0.996032178401947,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 1.0587,
+      "step": 150
+    },
+    {
+      "epoch": 0.014342058085335245,
+      "grad_norm": 0.9638823866844177,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 1.0386,
+      "step": 160
+    },
+    {
+      "epoch": 0.015238436715668699,
+      "grad_norm": 0.8804090619087219,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 1.0212,
+      "step": 170
+    },
+    {
+      "epoch": 0.01613481534600215,
+      "grad_norm": 1.0872684717178345,
+      "learning_rate": 2e-05,
+      "loss": 1.0481,
+      "step": 180
+    },
+    {
+      "epoch": 0.017031193976335603,
+      "grad_norm": 0.9900328516960144,
+      "learning_rate": 2e-05,
+      "loss": 1.0329,
+      "step": 190
+    },
+    {
+      "epoch": 0.017927572606669058,
+      "grad_norm": 0.9405505657196045,
+      "learning_rate": 2e-05,
+      "loss": 1.0436,
+      "step": 200
+    },
+    {
+      "epoch": 0.01882395123700251,
+      "grad_norm": 0.8467751145362854,
+      "learning_rate": 2e-05,
+      "loss": 1.0264,
+      "step": 210
+    },
+    {
+      "epoch": 0.01972032986733596,
+      "grad_norm": 0.8505803346633911,
+      "learning_rate": 2e-05,
+      "loss": 1.0329,
+      "step": 220
+    },
+    {
+      "epoch": 0.020616708497669417,
+      "grad_norm": 0.8671713471412659,
+      "learning_rate": 2e-05,
+      "loss": 1.0338,
+      "step": 230
+    },
+    {
+      "epoch": 0.02151308712800287,
+      "grad_norm": 0.7910956740379333,
+      "learning_rate": 2e-05,
+      "loss": 1.0336,
+      "step": 240
+    },
+    {
+      "epoch": 0.02240946575833632,
+      "grad_norm": 0.7766634225845337,
+      "learning_rate": 2e-05,
+      "loss": 1.0244,
+      "step": 250
+    },
+    {
+      "epoch": 0.023305844388669773,
+      "grad_norm": 0.7434602975845337,
+      "learning_rate": 2e-05,
+      "loss": 1.0163,
+      "step": 260
+    },
+    {
+      "epoch": 0.024202223019003228,
+      "grad_norm": 0.7649397850036621,
+      "learning_rate": 2e-05,
+      "loss": 0.9968,
+      "step": 270
+    },
+    {
+      "epoch": 0.02509860164933668,
+      "grad_norm": 0.8214126229286194,
+      "learning_rate": 2e-05,
+      "loss": 1.0132,
+      "step": 280
+    },
+    {
+      "epoch": 0.02599498027967013,
+      "grad_norm": 0.696774423122406,
+      "learning_rate": 2e-05,
+      "loss": 1.0159,
+      "step": 290
+    },
+    {
+      "epoch": 0.026891358910003587,
+      "grad_norm": 0.7053740620613098,
+      "learning_rate": 2e-05,
+      "loss": 1.0077,
+      "step": 300
+    },
+    {
+      "epoch": 0.02778773754033704,
+      "grad_norm": 0.7113314867019653,
+      "learning_rate": 2e-05,
+      "loss": 1.0155,
+      "step": 310
+    },
+    {
+      "epoch": 0.02868411617067049,
+      "grad_norm": 0.6863505244255066,
+      "learning_rate": 2e-05,
+      "loss": 1.0043,
+      "step": 320
+    },
+    {
+      "epoch": 0.029580494801003943,
+      "grad_norm": 0.6552082896232605,
+      "learning_rate": 2e-05,
+      "loss": 1.0083,
+      "step": 330
+    },
+    {
+      "epoch": 0.030476873431337398,
+      "grad_norm": 0.6568509936332703,
+      "learning_rate": 2e-05,
+      "loss": 1.0166,
+      "step": 340
+    },
+    {
+      "epoch": 0.03137325206167085,
+      "grad_norm": 0.7203536629676819,
+      "learning_rate": 2e-05,
+      "loss": 0.9978,
+      "step": 350
+    },
+    {
+      "epoch": 0.0322696306920043,
+      "grad_norm": 0.7241279482841492,
+      "learning_rate": 2e-05,
+      "loss": 0.9919,
+      "step": 360
+    },
+    {
+      "epoch": 0.03316600932233776,
+      "grad_norm": 0.6524237990379333,
+      "learning_rate": 2e-05,
+      "loss": 1.0067,
+      "step": 370
+    },
+    {
+      "epoch": 0.034062387952671205,
+      "grad_norm": 0.6274630427360535,
+      "learning_rate": 2e-05,
+      "loss": 1.0004,
+      "step": 380
+    },
+    {
+      "epoch": 0.03495876658300466,
+      "grad_norm": 0.6271491646766663,
+      "learning_rate": 2e-05,
+      "loss": 1.0133,
+      "step": 390
+    },
+    {
+      "epoch": 0.035855145213338116,
+      "grad_norm": 0.6838966012001038,
+      "learning_rate": 2e-05,
+      "loss": 1.0088,
+      "step": 400
+    },
+    {
+      "epoch": 0.036751523843671564,
+      "grad_norm": 0.6824681758880615,
+      "learning_rate": 2e-05,
+      "loss": 1.0162,
+      "step": 410
+    },
+    {
+      "epoch": 0.03764790247400502,
+      "grad_norm": 0.6277276277542114,
+      "learning_rate": 2e-05,
+      "loss": 1.0009,
+      "step": 420
+    },
+    {
+      "epoch": 0.038544281104338475,
+      "grad_norm": 0.6885059475898743,
+      "learning_rate": 2e-05,
+      "loss": 1.0131,
+      "step": 430
+    },
+    {
+      "epoch": 0.03944065973467192,
+      "grad_norm": 0.684141993522644,
+      "learning_rate": 2e-05,
+      "loss": 1.0026,
+      "step": 440
+    },
+    {
+      "epoch": 0.04033703836500538,
+      "grad_norm": 0.6428585648536682,
+      "learning_rate": 2e-05,
+      "loss": 1.0219,
+      "step": 450
+    },
+    {
+      "epoch": 0.041233416995338834,
+      "grad_norm": 0.6308277249336243,
+      "learning_rate": 2e-05,
+      "loss": 0.9994,
+      "step": 460
+    },
+    {
+      "epoch": 0.04212979562567228,
+      "grad_norm": 0.6679654121398926,
+      "learning_rate": 2e-05,
+      "loss": 0.9911,
+      "step": 470
+    },
+    {
+      "epoch": 0.04302617425600574,
+      "grad_norm": 0.5856204032897949,
+      "learning_rate": 2e-05,
+      "loss": 1.0133,
+      "step": 480
+    },
+    {
+      "epoch": 0.04392255288633919,
+      "grad_norm": 0.6006633639335632,
+      "learning_rate": 2e-05,
+      "loss": 0.9996,
+      "step": 490
+    },
+    {
+      "epoch": 0.04481893151667264,
+      "grad_norm": 0.6218353509902954,
+      "learning_rate": 2e-05,
+      "loss": 1.0088,
+      "step": 500
+    },
+    {
+      "epoch": 0.0457153101470061,
+      "grad_norm": 0.8754221796989441,
+      "learning_rate": 2e-05,
+      "loss": 0.9889,
+      "step": 510
+    },
+    {
+      "epoch": 0.046611688777339545,
+      "grad_norm": 0.5963256359100342,
+      "learning_rate": 2e-05,
+      "loss": 0.9883,
+      "step": 520
+    },
+    {
+      "epoch": 0.047508067407673,
+      "grad_norm": 0.6348252296447754,
+      "learning_rate": 2e-05,
+      "loss": 1.0094,
+      "step": 530
+    },
+    {
+      "epoch": 0.048404446038006456,
+      "grad_norm": 0.5878388285636902,
+      "learning_rate": 2e-05,
+      "loss": 1.0011,
+      "step": 540
+    },
+    {
+      "epoch": 0.049300824668339904,
+      "grad_norm": 0.5898565053939819,
+      "learning_rate": 2e-05,
+      "loss": 0.9948,
+      "step": 550
+    },
+    {
+      "epoch": 0.05019720329867336,
+      "grad_norm": 0.606599748134613,
+      "learning_rate": 2e-05,
+      "loss": 0.9931,
+      "step": 560
+    },
+    {
+      "epoch": 0.051093581929006815,
+      "grad_norm": 0.5964750051498413,
+      "learning_rate": 2e-05,
+      "loss": 0.9927,
+      "step": 570
+    },
+    {
+      "epoch": 0.05198996055934026,
+      "grad_norm": 0.582025408744812,
+      "learning_rate": 2e-05,
+      "loss": 0.9831,
+      "step": 580
+    },
+    {
+      "epoch": 0.05288633918967372,
+      "grad_norm": 0.5804903507232666,
+      "learning_rate": 2e-05,
+      "loss": 1.0006,
+      "step": 590
+    },
+    {
+      "epoch": 0.053782717820007174,
+      "grad_norm": 0.6151660680770874,
+      "learning_rate": 2e-05,
+      "loss": 0.9919,
+      "step": 600
+    },
+    {
+      "epoch": 0.05467909645034062,
+      "grad_norm": 0.5723623037338257,
+      "learning_rate": 2e-05,
+      "loss": 0.9987,
+      "step": 610
+    },
+    {
+      "epoch": 0.05557547508067408,
+      "grad_norm": 0.5697692036628723,
+      "learning_rate": 2e-05,
+      "loss": 0.9984,
+      "step": 620
+    },
+    {
+      "epoch": 0.05647185371100753,
+      "grad_norm": 0.5792081952095032,
+      "learning_rate": 2e-05,
+      "loss": 1.0001,
+      "step": 630
+    },
+    {
+      "epoch": 0.05736823234134098,
+      "grad_norm": 0.6127511858940125,
+      "learning_rate": 2e-05,
+      "loss": 1.0064,
+      "step": 640
+    },
+    {
+      "epoch": 0.05826461097167444,
+      "grad_norm": 0.554460346698761,
+      "learning_rate": 2e-05,
+      "loss": 0.9912,
+      "step": 650
+    },
+    {
+      "epoch": 0.059160989602007885,
+      "grad_norm": 0.5743119120597839,
+      "learning_rate": 2e-05,
+      "loss": 0.9892,
+      "step": 660
+    },
+    {
+      "epoch": 0.06005736823234134,
+      "grad_norm": 0.547473132610321,
+      "learning_rate": 2e-05,
+      "loss": 0.9689,
+      "step": 670
+    },
+    {
+      "epoch": 0.060953746862674796,
+      "grad_norm": 0.585006594657898,
+      "learning_rate": 2e-05,
+      "loss": 1.0037,
+      "step": 680
+    },
+    {
+      "epoch": 0.061850125493008244,
+      "grad_norm": 0.5973238348960876,
+      "learning_rate": 2e-05,
+      "loss": 0.9834,
+      "step": 690
+    },
+    {
+      "epoch": 0.0627465041233417,
+      "grad_norm": 0.569121778011322,
+      "learning_rate": 2e-05,
+      "loss": 0.9781,
+      "step": 700
+    },
+    {
+      "epoch": 0.06364288275367515,
+      "grad_norm": 0.5590016841888428,
+      "learning_rate": 2e-05,
+      "loss": 0.9872,
+      "step": 710
+    },
+    {
+      "epoch": 0.0645392613840086,
+      "grad_norm": 0.5477997660636902,
+      "learning_rate": 2e-05,
+      "loss": 1.0075,
+      "step": 720
+    },
+    {
+      "epoch": 0.06543564001434206,
+      "grad_norm": 0.5502426028251648,
+      "learning_rate": 2e-05,
+      "loss": 0.9898,
+      "step": 730
+    },
+    {
+      "epoch": 0.06633201864467551,
+      "grad_norm": 1.000443458557129,
+      "learning_rate": 2e-05,
+      "loss": 0.9911,
+      "step": 740
+    },
+    {
+      "epoch": 0.06722839727500897,
+      "grad_norm": 0.5668296813964844,
+      "learning_rate": 2e-05,
+      "loss": 0.9745,
+      "step": 750
+    },
+    {
+      "epoch": 0.06812477590534241,
+      "grad_norm": 0.5598611831665039,
+      "learning_rate": 2e-05,
+      "loss": 0.9811,
+      "step": 760
+    },
+    {
+      "epoch": 0.06902115453567587,
+      "grad_norm": 0.5525695085525513,
+      "learning_rate": 2e-05,
+      "loss": 0.9964,
+      "step": 770
+    },
+    {
+      "epoch": 0.06991753316600932,
+      "grad_norm": 0.6044715046882629,
+      "learning_rate": 2e-05,
+      "loss": 0.9738,
+      "step": 780
+    },
+    {
+      "epoch": 0.07081391179634278,
+      "grad_norm": 0.5423434972763062,
+      "learning_rate": 2e-05,
+      "loss": 0.9815,
+      "step": 790
+    },
+    {
+      "epoch": 0.07171029042667623,
+      "grad_norm": 0.5274398922920227,
+      "learning_rate": 2e-05,
+      "loss": 0.9593,
+      "step": 800
+    },
+    {
+      "epoch": 0.07260666905700969,
+      "grad_norm": 0.5719221830368042,
+      "learning_rate": 2e-05,
+      "loss": 0.9766,
+      "step": 810
+    },
+    {
+      "epoch": 0.07350304768734313,
+      "grad_norm": 0.5694926381111145,
+      "learning_rate": 2e-05,
+      "loss": 0.9808,
+      "step": 820
+    },
+    {
+      "epoch": 0.07439942631767658,
+      "grad_norm": 0.5297074317932129,
+      "learning_rate": 2e-05,
+      "loss": 0.9861,
+      "step": 830
+    },
+    {
+      "epoch": 0.07529580494801004,
+      "grad_norm": 0.5370926260948181,
+      "learning_rate": 2e-05,
+      "loss": 0.9817,
+      "step": 840
+    },
+    {
+      "epoch": 0.0761921835783435,
+      "grad_norm": 0.5671987533569336,
+      "learning_rate": 2e-05,
+      "loss": 0.9984,
+      "step": 850
+    },
+    {
+      "epoch": 0.07708856220867695,
+      "grad_norm": 0.5632290244102478,
+      "learning_rate": 2e-05,
+      "loss": 0.9796,
+      "step": 860
+    },
+    {
+      "epoch": 0.07798494083901039,
+      "grad_norm": 0.5388137102127075,
+      "learning_rate": 2e-05,
+      "loss": 0.9804,
+      "step": 870
+    },
+    {
+      "epoch": 0.07888131946934385,
+      "grad_norm": 0.5523011088371277,
+      "learning_rate": 2e-05,
+      "loss": 0.9844,
+      "step": 880
+    },
+    {
+      "epoch": 0.0797776980996773,
+      "grad_norm": 0.711898684501648,
+      "learning_rate": 2e-05,
+      "loss": 0.9751,
+      "step": 890
+    },
+    {
+      "epoch": 0.08067407673001076,
+      "grad_norm": 0.5636762380599976,
+      "learning_rate": 2e-05,
+      "loss": 0.9761,
+      "step": 900
+    },
+    {
+      "epoch": 0.08157045536034421,
+      "grad_norm": 0.572536826133728,
+      "learning_rate": 2e-05,
+      "loss": 0.9572,
+      "step": 910
+    },
+    {
+      "epoch": 0.08246683399067767,
+      "grad_norm": 0.5344299674034119,
+      "learning_rate": 2e-05,
+      "loss": 0.9826,
+      "step": 920
+    },
+    {
+      "epoch": 0.08336321262101111,
+      "grad_norm": 0.5394155383110046,
+      "learning_rate": 2e-05,
+      "loss": 0.9889,
+      "step": 930
+    },
+    {
+      "epoch": 0.08425959125134456,
+      "grad_norm": 0.5411435961723328,
+      "learning_rate": 2e-05,
+      "loss": 0.981,
+      "step": 940
+    },
+    {
+      "epoch": 0.08515596988167802,
+      "grad_norm": 0.5469105839729309,
+      "learning_rate": 2e-05,
+      "loss": 0.9907,
+      "step": 950
+    },
+    {
+      "epoch": 0.08605234851201148,
+      "grad_norm": 0.5600571036338806,
+      "learning_rate": 2e-05,
+      "loss": 0.9898,
+      "step": 960
+    },
+    {
+      "epoch": 0.08694872714234493,
+      "grad_norm": 0.5799705982208252,
+      "learning_rate": 2e-05,
+      "loss": 0.9707,
+      "step": 970
+    },
+    {
+      "epoch": 0.08784510577267839,
+      "grad_norm": 0.5643452405929565,
+      "learning_rate": 2e-05,
+      "loss": 0.9788,
+      "step": 980
+    },
+    {
+      "epoch": 0.08874148440301183,
+      "grad_norm": 0.5454543828964233,
+      "learning_rate": 2e-05,
+      "loss": 0.9758,
+      "step": 990
+    },
+    {
+      "epoch": 0.08963786303334528,
+      "grad_norm": 0.5381706357002258,
+      "learning_rate": 2e-05,
+      "loss": 0.9744,
+      "step": 1000
+    },
+    {
+      "epoch": 0.09053424166367874,
+      "grad_norm": 0.5545519590377808,
+      "learning_rate": 2e-05,
+      "loss": 0.9853,
+      "step": 1010
+    },
+    {
+      "epoch": 0.0914306202940122,
+      "grad_norm": 0.5297814011573792,
+      "learning_rate": 2e-05,
+      "loss": 0.9686,
+      "step": 1020
+    },
+    {
+      "epoch": 0.09232699892434565,
+      "grad_norm": 0.5557345747947693,
+      "learning_rate": 2e-05,
+      "loss": 0.9671,
+      "step": 1030
+    },
+    {
+      "epoch": 0.09322337755467909,
+      "grad_norm": 0.5732296705245972,
+      "learning_rate": 2e-05,
+      "loss": 0.978,
+      "step": 1040
+    },
+    {
+      "epoch": 0.09411975618501255,
+      "grad_norm": 0.5199934244155884,
+      "learning_rate": 2e-05,
+      "loss": 0.9891,
+      "step": 1050
+    },
+    {
+      "epoch": 0.095016134815346,
+      "grad_norm": 0.5543072819709778,
+      "learning_rate": 2e-05,
+      "loss": 0.988,
+      "step": 1060
+    },
+    {
+      "epoch": 0.09591251344567946,
+      "grad_norm": 0.5265071392059326,
+      "learning_rate": 2e-05,
+      "loss": 0.9666,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09680889207601291,
+      "grad_norm": 0.5349189639091492,
+      "learning_rate": 2e-05,
+      "loss": 0.9932,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09770527070634637,
+      "grad_norm": 0.6431905031204224,
+      "learning_rate": 2e-05,
+      "loss": 0.9833,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09860164933667981,
+      "grad_norm": 0.5542317628860474,
+      "learning_rate": 2e-05,
+      "loss": 0.9745,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09949802796701326,
+      "grad_norm": 0.6008753180503845,
+      "learning_rate": 2e-05,
+      "loss": 0.9847,
+      "step": 1110
+    },
+    {
+      "epoch": 0.10039440659734672,
+      "grad_norm": 0.5337353348731995,
+      "learning_rate": 2e-05,
+      "loss": 0.9848,
+      "step": 1120
+    },
+    {
+      "epoch": 0.10129078522768017,
+      "grad_norm": 0.5480239391326904,
+      "learning_rate": 2e-05,
+      "loss": 0.9868,
+      "step": 1130
+    },
+    {
+      "epoch": 0.10218716385801363,
+      "grad_norm": 0.546202540397644,
+      "learning_rate": 2e-05,
+      "loss": 0.9688,
+      "step": 1140
+    },
+    {
+      "epoch": 0.10308354248834707,
+      "grad_norm": 0.5187017917633057,
+      "learning_rate": 2e-05,
+      "loss": 0.9715,
+      "step": 1150
+    },
+    {
+      "epoch": 0.10397992111868053,
+      "grad_norm": 0.6519541144371033,
+      "learning_rate": 2e-05,
+      "loss": 0.9737,
+      "step": 1160
+    },
+    {
+      "epoch": 0.10487629974901398,
+      "grad_norm": 0.5310616493225098,
+      "learning_rate": 2e-05,
+      "loss": 0.9833,
+      "step": 1170
+    },
+    {
+      "epoch": 0.10577267837934744,
+      "grad_norm": 0.5454709529876709,
+      "learning_rate": 2e-05,
+      "loss": 0.95,
+      "step": 1180
+    },
+    {
+      "epoch": 0.10666905700968089,
+      "grad_norm": 0.5410040020942688,
+      "learning_rate": 2e-05,
+      "loss": 0.9826,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10756543564001435,
+      "grad_norm": 0.5235140323638916,
+      "learning_rate": 2e-05,
+      "loss": 0.9596,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10846181427034779,
+      "grad_norm": 0.6987593173980713,
+      "learning_rate": 2e-05,
+      "loss": 0.9721,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10935819290068124,
+      "grad_norm": 0.5288031697273254,
+      "learning_rate": 2e-05,
+      "loss": 0.9745,
+      "step": 1220
+    },
+    {
+      "epoch": 0.1102545715310147,
+      "grad_norm": 0.5067135095596313,
+      "learning_rate": 2e-05,
+      "loss": 0.9657,
+      "step": 1230
+    },
+    {
+      "epoch": 0.11115095016134816,
+      "grad_norm": 0.5409311056137085,
+      "learning_rate": 2e-05,
+      "loss": 0.9643,
+      "step": 1240
+    },
+    {
+      "epoch": 0.11204732879168161,
+      "grad_norm": 0.5261286497116089,
+      "learning_rate": 2e-05,
+      "loss": 0.9825,
+      "step": 1250
+    },
+    {
+      "epoch": 0.11294370742201507,
+      "grad_norm": 0.5786731839179993,
+      "learning_rate": 2e-05,
+      "loss": 0.9754,
+      "step": 1260
+    },
+    {
+      "epoch": 0.11384008605234851,
+      "grad_norm": 0.5430377125740051,
+      "learning_rate": 2e-05,
+      "loss": 0.9638,
+      "step": 1270
+    },
+    {
+      "epoch": 0.11473646468268196,
+      "grad_norm": 0.5725428462028503,
+      "learning_rate": 2e-05,
+      "loss": 0.973,
+      "step": 1280
+    },
+    {
+      "epoch": 0.11563284331301542,
+      "grad_norm": 0.535925030708313,
+      "learning_rate": 2e-05,
+      "loss": 0.9892,
+      "step": 1290
+    },
+    {
+      "epoch": 0.11652922194334887,
+      "grad_norm": 0.5398180484771729,
+      "learning_rate": 2e-05,
+      "loss": 0.9715,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11742560057368233,
+      "grad_norm": 0.5545116662979126,
+      "learning_rate": 2e-05,
+      "loss": 0.964,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11832197920401577,
+      "grad_norm": 0.5387099385261536,
+      "learning_rate": 2e-05,
+      "loss": 0.9682,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11921835783434923,
+      "grad_norm": 0.5465854406356812,
+      "learning_rate": 2e-05,
+      "loss": 0.9663,
+      "step": 1330
+    },
+    {
+      "epoch": 0.12011473646468268,
+      "grad_norm": 0.8989889621734619,
+      "learning_rate": 2e-05,
+      "loss": 0.9666,
+      "step": 1340
+    },
+    {
+      "epoch": 0.12101111509501614,
+      "grad_norm": 0.5535284280776978,
+      "learning_rate": 2e-05,
+      "loss": 0.9876,
+      "step": 1350
+    },
+    {
+      "epoch": 0.12190749372534959,
+      "grad_norm": 0.5118777751922607,
+      "learning_rate": 2e-05,
+      "loss": 0.9635,
+      "step": 1360
+    },
+    {
+      "epoch": 0.12280387235568305,
+      "grad_norm": 0.5333411693572998,
+      "learning_rate": 2e-05,
+      "loss": 0.9697,
+      "step": 1370
+    },
+    {
+      "epoch": 0.12370025098601649,
+      "grad_norm": 0.5208771824836731,
+      "learning_rate": 2e-05,
+      "loss": 0.9632,
+      "step": 1380
+    },
+    {
+      "epoch": 0.12459662961634994,
+      "grad_norm": 0.5455602407455444,
+      "learning_rate": 2e-05,
+      "loss": 0.9663,
+      "step": 1390
+    },
+    {
+      "epoch": 0.1254930082466834,
+      "grad_norm": 0.5906007885932922,
+      "learning_rate": 2e-05,
+      "loss": 0.9549,
+      "step": 1400
+    },
+    {
+      "epoch": 0.12638938687701684,
+      "grad_norm": 0.799707293510437,
+      "learning_rate": 2e-05,
+      "loss": 0.9532,
+      "step": 1410
+    },
+    {
+      "epoch": 0.1272857655073503,
+      "grad_norm": 0.5657065510749817,
+      "learning_rate": 2e-05,
+      "loss": 0.9875,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12818214413768375,
+      "grad_norm": 0.5292009115219116,
+      "learning_rate": 2e-05,
+      "loss": 0.9709,
+      "step": 1430
+    },
+    {
+      "epoch": 0.1290785227680172,
+      "grad_norm": 0.522810161113739,
+      "learning_rate": 2e-05,
+      "loss": 0.9608,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12997490139835066,
+      "grad_norm": 0.5288273096084595,
+      "learning_rate": 2e-05,
+      "loss": 0.9664,
+      "step": 1450
+    },
+    {
+      "epoch": 0.13087128002868412,
+      "grad_norm": 0.5530571341514587,
+      "learning_rate": 2e-05,
+      "loss": 0.9716,
+      "step": 1460
+    },
+    {
+      "epoch": 0.13176765865901757,
+      "grad_norm": 0.5197150707244873,
+      "learning_rate": 2e-05,
+      "loss": 0.9597,
+      "step": 1470
+    },
+    {
+      "epoch": 0.13266403728935103,
+      "grad_norm": 0.5461387634277344,
+      "learning_rate": 2e-05,
+      "loss": 0.9605,
+      "step": 1480
+    },
+    {
+      "epoch": 0.13356041591968448,
+      "grad_norm": 0.5766053795814514,
+      "learning_rate": 2e-05,
+      "loss": 0.9648,
+      "step": 1490
+    },
+    {
+      "epoch": 0.13445679455001794,
+      "grad_norm": 0.5547395348548889,
+      "learning_rate": 2e-05,
+      "loss": 0.9614,
+      "step": 1500
+    },
+    {
+      "epoch": 0.1353531731803514,
+      "grad_norm": 0.5190296769142151,
+      "learning_rate": 2e-05,
+      "loss": 0.9612,
+      "step": 1510
+    },
+    {
+      "epoch": 0.13624955181068482,
+      "grad_norm": 0.535395622253418,
+      "learning_rate": 2e-05,
+      "loss": 0.9588,
+      "step": 1520
+    },
+    {
+      "epoch": 0.13714593044101828,
+      "grad_norm": 0.5282490253448486,
+      "learning_rate": 2e-05,
+      "loss": 0.9653,
+      "step": 1530
+    },
+    {
+      "epoch": 0.13804230907135173,
+      "grad_norm": 0.5242656469345093,
+      "learning_rate": 2e-05,
+      "loss": 0.9631,
+      "step": 1540
+    },
+    {
+      "epoch": 0.1389386877016852,
+      "grad_norm": 0.5125828385353088,
+      "learning_rate": 2e-05,
+      "loss": 0.963,
+      "step": 1550
+    },
+    {
+      "epoch": 0.13983506633201864,
+      "grad_norm": 0.5051111578941345,
+      "learning_rate": 2e-05,
+      "loss": 0.9701,
+      "step": 1560
+    },
+    {
+      "epoch": 0.1407314449623521,
+      "grad_norm": 0.5633049607276917,
+      "learning_rate": 2e-05,
+      "loss": 0.9717,
+      "step": 1570
+    },
+    {
+      "epoch": 0.14162782359268555,
+      "grad_norm": 0.5364615321159363,
+      "learning_rate": 2e-05,
+      "loss": 0.9673,
+      "step": 1580
+    },
+    {
+      "epoch": 0.142524202223019,
+      "grad_norm": 0.5263501405715942,
+      "learning_rate": 2e-05,
+      "loss": 0.9646,
+      "step": 1590
+    },
+    {
+      "epoch": 0.14342058085335246,
+      "grad_norm": 0.5146853923797607,
+      "learning_rate": 2e-05,
+      "loss": 0.965,
+      "step": 1600
+    },
+    {
+      "epoch": 0.14431695948368592,
+      "grad_norm": 0.5201026797294617,
+      "learning_rate": 2e-05,
+      "loss": 0.9612,
+      "step": 1610
+    },
+    {
+      "epoch": 0.14521333811401937,
+      "grad_norm": 0.6016380190849304,
+      "learning_rate": 2e-05,
+      "loss": 0.9665,
+      "step": 1620
+    },
+    {
+      "epoch": 0.1461097167443528,
+      "grad_norm": 0.501792311668396,
+      "learning_rate": 2e-05,
+      "loss": 0.9566,
+      "step": 1630
+    },
+    {
+      "epoch": 0.14700609537468626,
+      "grad_norm": 0.5101803541183472,
+      "learning_rate": 2e-05,
+      "loss": 0.9566,
+      "step": 1640
+    },
+    {
+      "epoch": 0.1479024740050197,
+      "grad_norm": 0.5033829212188721,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 1650
+    },
+    {
+      "epoch": 0.14879885263535317,
+      "grad_norm": 0.5989149212837219,
+      "learning_rate": 2e-05,
+      "loss": 0.9659,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14969523126568662,
+      "grad_norm": 0.4935537874698639,
+      "learning_rate": 2e-05,
+      "loss": 0.9477,
+      "step": 1670
+    },
+    {
+      "epoch": 0.15059160989602008,
+      "grad_norm": 0.5393995046615601,
+      "learning_rate": 2e-05,
+      "loss": 0.967,
+      "step": 1680
+    },
+    {
+      "epoch": 0.15148798852635353,
+      "grad_norm": 0.5287318229675293,
+      "learning_rate": 2e-05,
+      "loss": 0.959,
+      "step": 1690
+    },
+    {
+      "epoch": 0.152384367156687,
+      "grad_norm": 0.5194284319877625,
+      "learning_rate": 2e-05,
+      "loss": 0.9624,
+      "step": 1700
+    },
+    {
+      "epoch": 0.15328074578702044,
+      "grad_norm": 0.6787471771240234,
+      "learning_rate": 2e-05,
+      "loss": 0.986,
+      "step": 1710
+    },
+    {
+      "epoch": 0.1541771244173539,
+      "grad_norm": 0.5294409394264221,
+      "learning_rate": 2e-05,
+      "loss": 0.9664,
+      "step": 1720
+    },
+    {
+      "epoch": 0.15507350304768736,
+      "grad_norm": 0.5209848284721375,
+      "learning_rate": 2e-05,
+      "loss": 0.9703,
+      "step": 1730
+    },
+    {
+      "epoch": 0.15596988167802078,
+      "grad_norm": 0.5797594785690308,
+      "learning_rate": 2e-05,
+      "loss": 0.9517,
+      "step": 1740
+    },
+    {
+      "epoch": 0.15686626030835424,
+      "grad_norm": 1.0721763372421265,
+      "learning_rate": 2e-05,
+      "loss": 0.9648,
+      "step": 1750
+    },
+    {
+      "epoch": 0.1577626389386877,
+      "grad_norm": 0.5732969641685486,
+      "learning_rate": 2e-05,
+      "loss": 0.9534,
+      "step": 1760
+    },
+    {
+      "epoch": 0.15865901756902115,
+      "grad_norm": 0.5131897330284119,
+      "learning_rate": 2e-05,
+      "loss": 0.9562,
+      "step": 1770
+    },
+    {
+      "epoch": 0.1595553961993546,
+      "grad_norm": 0.5333603024482727,
+      "learning_rate": 2e-05,
+      "loss": 0.9705,
+      "step": 1780
+    },
+    {
+      "epoch": 0.16045177482968806,
+      "grad_norm": 0.5011164546012878,
+      "learning_rate": 2e-05,
+      "loss": 0.9485,
+      "step": 1790
+    },
+    {
+      "epoch": 0.16134815346002152,
+      "grad_norm": 0.5204216241836548,
+      "learning_rate": 2e-05,
+      "loss": 0.9718,
+      "step": 1800
+    },
+    {
+      "epoch": 0.16224453209035497,
+      "grad_norm": 0.5532276034355164,
+      "learning_rate": 2e-05,
+      "loss": 0.9558,
+      "step": 1810
+    },
+    {
+      "epoch": 0.16314091072068843,
+      "grad_norm": 0.5219212174415588,
+      "learning_rate": 2e-05,
+      "loss": 0.9624,
+      "step": 1820
+    },
+    {
+      "epoch": 0.16403728935102188,
+      "grad_norm": 0.5574737787246704,
+      "learning_rate": 2e-05,
+      "loss": 0.9615,
+      "step": 1830
+    },
+    {
+      "epoch": 0.16493366798135534,
+      "grad_norm": 0.5201219320297241,
+      "learning_rate": 2e-05,
+      "loss": 0.9502,
+      "step": 1840
+    },
+    {
+      "epoch": 0.16583004661168876,
+      "grad_norm": 0.5013785362243652,
+      "learning_rate": 2e-05,
+      "loss": 0.958,
+      "step": 1850
+    },
+    {
+      "epoch": 0.16672642524202222,
+      "grad_norm": 0.5224004983901978,
+      "learning_rate": 2e-05,
+      "loss": 0.9534,
+      "step": 1860
+    },
+    {
+      "epoch": 0.16762280387235567,
+      "grad_norm": 0.5411056876182556,
+      "learning_rate": 2e-05,
+      "loss": 0.9648,
+      "step": 1870
+    },
+    {
+      "epoch": 0.16851918250268913,
+      "grad_norm": 0.5216169953346252,
+      "learning_rate": 2e-05,
+      "loss": 0.9478,
+      "step": 1880
+    },
+    {
+      "epoch": 0.16941556113302259,
+      "grad_norm": 0.5166903138160706,
+      "learning_rate": 2e-05,
+      "loss": 0.9606,
+      "step": 1890
+    },
+    {
+      "epoch": 0.17031193976335604,
+      "grad_norm": 0.5232516527175903,
+      "learning_rate": 2e-05,
+      "loss": 0.9547,
+      "step": 1900
+    },
+    {
+      "epoch": 0.1712083183936895,
+      "grad_norm": 0.518708348274231,
+      "learning_rate": 2e-05,
+      "loss": 0.9608,
+      "step": 1910
+    },
+    {
+      "epoch": 0.17210469702402295,
+      "grad_norm": 0.5260992646217346,
+      "learning_rate": 2e-05,
+      "loss": 0.9226,
+      "step": 1920
+    },
+    {
+      "epoch": 0.1730010756543564,
+      "grad_norm": 0.5175358653068542,
+      "learning_rate": 2e-05,
+      "loss": 0.9557,
+      "step": 1930
+    },
+    {
+      "epoch": 0.17389745428468986,
+      "grad_norm": 0.514107882976532,
+      "learning_rate": 2e-05,
+      "loss": 0.9495,
+      "step": 1940
+    },
+    {
+      "epoch": 0.17479383291502332,
+      "grad_norm": 0.5307929515838623,
+      "learning_rate": 2e-05,
+      "loss": 0.976,
+      "step": 1950
+    },
+    {
+      "epoch": 0.17569021154535677,
+      "grad_norm": 0.5089200735092163,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 1960
+    },
+    {
+      "epoch": 0.1765865901756902,
+      "grad_norm": 0.5342289805412292,
+      "learning_rate": 2e-05,
+      "loss": 0.9501,
+      "step": 1970
+    },
+    {
+      "epoch": 0.17748296880602366,
+      "grad_norm": 0.518286406993866,
+      "learning_rate": 2e-05,
+      "loss": 0.9445,
+      "step": 1980
+    },
+    {
+      "epoch": 0.1783793474363571,
+      "grad_norm": 0.5307959318161011,
+      "learning_rate": 2e-05,
+      "loss": 0.9529,
+      "step": 1990
+    },
+    {
+      "epoch": 0.17927572606669057,
+      "grad_norm": 0.520534873008728,
+      "learning_rate": 2e-05,
+      "loss": 0.9571,
+      "step": 2000
+    },
+    {
+      "epoch": 0.18017210469702402,
+      "grad_norm": 0.5103996396064758,
+      "learning_rate": 2e-05,
+      "loss": 0.9527,
+      "step": 2010
+    },
+    {
+      "epoch": 0.18106848332735748,
+      "grad_norm": 0.49629834294319153,
+      "learning_rate": 2e-05,
+      "loss": 0.9545,
+      "step": 2020
+    },
+    {
+      "epoch": 0.18196486195769093,
+      "grad_norm": 0.5511291027069092,
+      "learning_rate": 2e-05,
+      "loss": 0.9644,
+      "step": 2030
+    },
+    {
+      "epoch": 0.1828612405880244,
+      "grad_norm": 0.5241140127182007,
+      "learning_rate": 2e-05,
+      "loss": 0.9597,
+      "step": 2040
+    },
+    {
+      "epoch": 0.18375761921835784,
+      "grad_norm": 0.5620214939117432,
+      "learning_rate": 2e-05,
+      "loss": 0.947,
+      "step": 2050
+    },
+    {
+      "epoch": 0.1846539978486913,
+      "grad_norm": 0.5207156538963318,
+      "learning_rate": 2e-05,
+      "loss": 0.953,
+      "step": 2060
+    },
+    {
+      "epoch": 0.18555037647902475,
+      "grad_norm": 0.5182037353515625,
+      "learning_rate": 2e-05,
+      "loss": 0.9543,
+      "step": 2070
+    },
+    {
+      "epoch": 0.18644675510935818,
+      "grad_norm": 0.5094236731529236,
+      "learning_rate": 2e-05,
+      "loss": 0.9642,
+      "step": 2080
+    },
+    {
+      "epoch": 0.18734313373969164,
+      "grad_norm": 0.5734402537345886,
+      "learning_rate": 2e-05,
+      "loss": 0.9595,
+      "step": 2090
+    },
+    {
+      "epoch": 0.1882395123700251,
+      "grad_norm": 0.5211153626441956,
+      "learning_rate": 2e-05,
+      "loss": 0.9598,
+      "step": 2100
+    },
+    {
+      "epoch": 0.18913589100035855,
+      "grad_norm": 0.5086119174957275,
+      "learning_rate": 2e-05,
+      "loss": 0.9654,
+      "step": 2110
+    },
+    {
+      "epoch": 0.190032269630692,
+      "grad_norm": 0.5247552394866943,
+      "learning_rate": 2e-05,
+      "loss": 0.9442,
+      "step": 2120
+    },
+    {
+      "epoch": 0.19092864826102546,
+      "grad_norm": 0.5156105160713196,
+      "learning_rate": 2e-05,
+      "loss": 0.9611,
+      "step": 2130
+    },
+    {
+      "epoch": 0.1918250268913589,
+      "grad_norm": 0.5329754948616028,
+      "learning_rate": 2e-05,
+      "loss": 0.9661,
+      "step": 2140
+    },
+    {
+      "epoch": 0.19272140552169237,
+      "grad_norm": 0.6516852974891663,
+      "learning_rate": 2e-05,
+      "loss": 0.9599,
+      "step": 2150
+    },
+    {
+      "epoch": 0.19361778415202582,
+      "grad_norm": 0.540736734867096,
+      "learning_rate": 2e-05,
+      "loss": 0.9544,
+      "step": 2160
+    },
+    {
+      "epoch": 0.19451416278235928,
+      "grad_norm": 0.5110966563224792,
+      "learning_rate": 2e-05,
+      "loss": 0.9474,
+      "step": 2170
+    },
+    {
+      "epoch": 0.19541054141269273,
+      "grad_norm": 0.5426011085510254,
+      "learning_rate": 2e-05,
+      "loss": 0.9459,
+      "step": 2180
+    },
+    {
+      "epoch": 0.19630692004302616,
+      "grad_norm": 0.5007818341255188,
+      "learning_rate": 2e-05,
+      "loss": 0.9516,
+      "step": 2190
+    },
+    {
+      "epoch": 0.19720329867335962,
+      "grad_norm": 0.5127503871917725,
+      "learning_rate": 2e-05,
+      "loss": 0.9564,
+      "step": 2200
+    },
+    {
+      "epoch": 0.19809967730369307,
+      "grad_norm": 0.5129677653312683,
+      "learning_rate": 2e-05,
+      "loss": 0.9684,
+      "step": 2210
+    },
+    {
+      "epoch": 0.19899605593402653,
+      "grad_norm": 0.5347315073013306,
+      "learning_rate": 2e-05,
+      "loss": 0.9603,
+      "step": 2220
+    },
+    {
+      "epoch": 0.19989243456435998,
+      "grad_norm": 0.5020232796669006,
+      "learning_rate": 2e-05,
+      "loss": 0.9534,
+      "step": 2230
+    },
+    {
+      "epoch": 0.20078881319469344,
+      "grad_norm": 0.5001387000083923,
+      "learning_rate": 2e-05,
+      "loss": 0.945,
+      "step": 2240
+    },
+    {
+      "epoch": 0.2016851918250269,
+      "grad_norm": 0.5137271881103516,
+      "learning_rate": 2e-05,
+      "loss": 0.97,
+      "step": 2250
+    },
+    {
+      "epoch": 0.20258157045536035,
+      "grad_norm": 0.5089419484138489,
+      "learning_rate": 2e-05,
+      "loss": 0.9376,
+      "step": 2260
+    },
+    {
+      "epoch": 0.2034779490856938,
+      "grad_norm": 0.5090593099594116,
+      "learning_rate": 2e-05,
+      "loss": 0.9501,
+      "step": 2270
+    },
+    {
+      "epoch": 0.20437432771602726,
+      "grad_norm": 0.5081882476806641,
+      "learning_rate": 2e-05,
+      "loss": 0.934,
+      "step": 2280
+    },
+    {
+      "epoch": 0.20527070634636072,
+      "grad_norm": 0.6888887882232666,
+      "learning_rate": 2e-05,
+      "loss": 0.9337,
+      "step": 2290
+    },
+    {
+      "epoch": 0.20616708497669414,
+      "grad_norm": 0.625417947769165,
+      "learning_rate": 2e-05,
+      "loss": 0.9736,
+      "step": 2300
+    },
+    {
+      "epoch": 0.2070634636070276,
+      "grad_norm": 0.5230647325515747,
+      "learning_rate": 2e-05,
+      "loss": 0.9652,
+      "step": 2310
+    },
+    {
+      "epoch": 0.20795984223736105,
+      "grad_norm": 0.49617475271224976,
+      "learning_rate": 2e-05,
+      "loss": 0.9383,
+      "step": 2320
+    },
+    {
+      "epoch": 0.2088562208676945,
+      "grad_norm": 0.5560393929481506,
+      "learning_rate": 2e-05,
+      "loss": 0.9555,
+      "step": 2330
+    },
+    {
+      "epoch": 0.20975259949802796,
+      "grad_norm": 0.5088238716125488,
+      "learning_rate": 2e-05,
+      "loss": 0.9615,
+      "step": 2340
+    },
+    {
+      "epoch": 0.21064897812836142,
+      "grad_norm": 0.50723797082901,
+      "learning_rate": 2e-05,
+      "loss": 0.9617,
+      "step": 2350
+    },
+    {
+      "epoch": 0.21154535675869487,
+      "grad_norm": 0.5106563568115234,
+      "learning_rate": 2e-05,
+      "loss": 0.9273,
+      "step": 2360
+    },
+    {
+      "epoch": 0.21244173538902833,
+      "grad_norm": 0.5245212912559509,
+      "learning_rate": 2e-05,
+      "loss": 0.945,
+      "step": 2370
+    },
+    {
+      "epoch": 0.21333811401936179,
+      "grad_norm": 0.5194502472877502,
+      "learning_rate": 2e-05,
+      "loss": 0.957,
+      "step": 2380
+    },
+    {
+      "epoch": 0.21423449264969524,
+      "grad_norm": 0.5218809247016907,
+      "learning_rate": 2e-05,
+      "loss": 0.9587,
+      "step": 2390
+    },
+    {
+      "epoch": 0.2151308712800287,
+      "grad_norm": 0.5076110363006592,
+      "learning_rate": 2e-05,
+      "loss": 0.9501,
+      "step": 2400
+    },
+    {
+      "epoch": 0.21602724991036212,
+      "grad_norm": 0.5114862322807312,
+      "learning_rate": 2e-05,
+      "loss": 0.9571,
+      "step": 2410
+    },
+    {
+      "epoch": 0.21692362854069558,
+      "grad_norm": 0.5061441659927368,
+      "learning_rate": 2e-05,
+      "loss": 0.9602,
+      "step": 2420
+    },
+    {
+      "epoch": 0.21782000717102903,
+      "grad_norm": 0.48866280913352966,
+      "learning_rate": 2e-05,
+      "loss": 0.9486,
+      "step": 2430
+    },
+    {
+      "epoch": 0.2187163858013625,
+      "grad_norm": 0.5101744532585144,
+      "learning_rate": 2e-05,
+      "loss": 0.9526,
+      "step": 2440
+    },
+    {
+      "epoch": 0.21961276443169594,
+      "grad_norm": 0.5281191468238831,
+      "learning_rate": 2e-05,
+      "loss": 0.9414,
+      "step": 2450
+    },
+    {
+      "epoch": 0.2205091430620294,
+      "grad_norm": 0.5058172345161438,
+      "learning_rate": 2e-05,
+      "loss": 0.9682,
+      "step": 2460
+    },
+    {
+      "epoch": 0.22140552169236286,
+      "grad_norm": 0.5281968712806702,
+      "learning_rate": 2e-05,
+      "loss": 0.9761,
+      "step": 2470
+    },
+    {
+      "epoch": 0.2223019003226963,
+      "grad_norm": 0.5069180130958557,
+      "learning_rate": 2e-05,
+      "loss": 0.9486,
+      "step": 2480
+    },
+    {
+      "epoch": 0.22319827895302977,
+      "grad_norm": 0.5250263810157776,
+      "learning_rate": 2e-05,
+      "loss": 0.9447,
+      "step": 2490
+    },
+    {
+      "epoch": 0.22409465758336322,
+      "grad_norm": 0.5200939178466797,
+      "learning_rate": 2e-05,
+      "loss": 0.9402,
+      "step": 2500
+    },
+    {
+      "epoch": 0.22499103621369668,
+      "grad_norm": 0.5146573781967163,
+      "learning_rate": 2e-05,
+      "loss": 0.9526,
+      "step": 2510
+    },
+    {
+      "epoch": 0.22588741484403013,
+      "grad_norm": 0.5090529918670654,
+      "learning_rate": 2e-05,
+      "loss": 0.9449,
+      "step": 2520
+    },
+    {
+      "epoch": 0.22678379347436356,
+      "grad_norm": 0.5018084645271301,
+      "learning_rate": 2e-05,
+      "loss": 0.9513,
+      "step": 2530
+    },
+    {
+      "epoch": 0.22768017210469702,
+      "grad_norm": 0.5566006302833557,
+      "learning_rate": 2e-05,
+      "loss": 0.9455,
+      "step": 2540
+    },
+    {
+      "epoch": 0.22857655073503047,
+      "grad_norm": 0.5043192505836487,
+      "learning_rate": 1.997261400182901e-05,
+      "loss": 0.9168,
+      "step": 2550
+    },
+    {
+      "epoch": 0.22947292936536393,
+      "grad_norm": 0.5040760040283203,
+      "learning_rate": 1.9805800746810423e-05,
+      "loss": 0.9495,
+      "step": 2560
+    },
+    {
+      "epoch": 0.23036930799569738,
+      "grad_norm": 0.49980929493904114,
+      "learning_rate": 1.9489934220110274e-05,
+      "loss": 0.9421,
+      "step": 2570
+    },
+    {
+      "epoch": 0.23126568662603084,
+      "grad_norm": 0.504512369632721,
+      "learning_rate": 1.9029840364820392e-05,
+      "loss": 0.9519,
+      "step": 2580
+    },
+    {
+      "epoch": 0.2321620652563643,
+      "grad_norm": 0.48759084939956665,
+      "learning_rate": 1.8432548690479908e-05,
+      "loss": 0.9187,
+      "step": 2590
+    },
+    {
+      "epoch": 0.23305844388669775,
+      "grad_norm": 0.5166524052619934,
+      "learning_rate": 1.7707184873239772e-05,
+      "loss": 0.9465,
+      "step": 2600
+    },
+    {
+      "epoch": 0.2339548225170312,
+      "grad_norm": 0.49367469549179077,
+      "learning_rate": 1.6864831329902565e-05,
+      "loss": 0.9534,
+      "step": 2610
+    },
+    {
+      "epoch": 0.23485120114736466,
+      "grad_norm": 0.4911825656890869,
+      "learning_rate": 1.591835789604667e-05,
+      "loss": 0.9365,
+      "step": 2620
+    },
+    {
+      "epoch": 0.2357475797776981,
+      "grad_norm": 0.49763816595077515,
+      "learning_rate": 1.4882225195206459e-05,
+      "loss": 0.9476,
+      "step": 2630
+    },
+    {
+      "epoch": 0.23664395840803154,
+      "grad_norm": 0.4919648766517639,
+      "learning_rate": 1.3772263703317695e-05,
+      "loss": 0.9418,
+      "step": 2640
+    },
+    {
+      "epoch": 0.237540337038365,
+      "grad_norm": 0.49768131971359253,
+      "learning_rate": 1.2605431883975547e-05,
+      "loss": 0.9498,
+      "step": 2650
+    },
+    {
+      "epoch": 0.23843671566869845,
+      "grad_norm": 0.6019100546836853,
+      "learning_rate": 1.1399557089817574e-05,
+      "loss": 0.925,
+      "step": 2660
+    },
+    {
+      "epoch": 0.2393330942990319,
+      "grad_norm": 0.5030861496925354,
+      "learning_rate": 1.0173063188650463e-05,
+      "loss": 0.9592,
+      "step": 2670
+    },
+    {
+      "epoch": 0.24022947292936536,
+      "grad_norm": 0.5112511515617371,
+      "learning_rate": 8.944689075764396e-06,
+      "loss": 0.9319,
+      "step": 2680
+    },
+    {
+      "epoch": 0.24112585155969882,
+      "grad_norm": 0.49274057149887085,
+      "learning_rate": 7.73320237312362e-06,
+      "loss": 0.9464,
+      "step": 2690
+    },
+    {
+      "epoch": 0.24202223019003227,
+      "grad_norm": 0.49516919255256653,
+      "learning_rate": 6.557112689658629e-06,
+      "loss": 0.9257,
+      "step": 2700
+    },
+    {
+      "epoch": 0.24291860882036573,
+      "grad_norm": 0.4888593554496765,
+      "learning_rate": 5.434388823591213e-06,
+      "loss": 0.9491,
+      "step": 2710
+    },
+    {
+      "epoch": 0.24381498745069918,
+      "grad_norm": 0.4862637221813202,
+      "learning_rate": 4.382184227495424e-06,
+      "loss": 0.9357,
+      "step": 2720
+    },
+    {
+      "epoch": 0.24471136608103264,
+      "grad_norm": 0.4957457184791565,
+      "learning_rate": 3.4165749305560203e-06,
+      "loss": 0.9352,
+      "step": 2730
+    },
+    {
+      "epoch": 0.2456077447113661,
+      "grad_norm": 0.46941837668418884,
+      "learning_rate": 2.5523139221595556e-06,
+      "loss": 0.9424,
+      "step": 2740
+    },
+    {
+      "epoch": 0.24650412334169952,
+      "grad_norm": 0.4843917489051819,
+      "learning_rate": 1.8026057494499821e-06,
+      "loss": 0.9466,
+      "step": 2750
+    },
+    {
+      "epoch": 0.24740050197203298,
+      "grad_norm": 0.49358218908309937,
+      "learning_rate": 1.178904772643223e-06,
+      "loss": 0.956,
+      "step": 2760
+    },
+    {
+      "epoch": 0.24829688060236643,
+      "grad_norm": 0.4765986502170563,
+      "learning_rate": 6.90740160441754e-07,
+      "loss": 0.9211,
+      "step": 2770
+    },
+    {
+      "epoch": 0.2491932592326999,
+      "grad_norm": 0.4660724401473999,
+      "learning_rate": 3.45570299343632e-07,
+      "loss": 0.9277,
+      "step": 2780
+    },
+    {
+      "epoch": 0.25008963786303334,
+      "grad_norm": 0.4763626456260681,
+      "learning_rate": 1.486688412425152e-07,
+      "loss": 0.9296,
+      "step": 2790
+    },
+    {
+      "epoch": 0.25071710290426674,
+      "step": 2797,
+      "total_flos": 1.2485883762455871e+19,
+      "train_loss": 0.08493010026879254,
+      "train_runtime": 2880.0845,
+      "train_samples_per_second": 11.654,
+      "train_steps_per_second": 0.971
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2797,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 2543,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2485883762455871e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g1_blog_post/250M/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dc526f45c908ca37b7193afc04053ac3f000de2c9a6c9cc5d5b008682dccade
+size 9041