Add files using upload-large-folder tool

g1_0_baseline_original_code/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.25118951431905917,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.08442884704911598,
+    "train_runtime": 2296.0802,
+    "train_samples_per_second": 14.623,
+    "train_steps_per_second": 1.219
+}

g1_0_baseline_original_code/250M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_0_baseline_original_code/250M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_0_baseline_original_code/250M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_0_baseline_original_code/250M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_0_baseline_original_code/250M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_0_baseline_original_code/250M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_0_baseline_original_code/250M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.25118951431905917,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.08442884704911598,
+    "train_runtime": 2296.0802,
+    "train_samples_per_second": 14.623,
+    "train_steps_per_second": 1.219
+}

g1_0_baseline_original_code/250M/trainer_log.jsonl

@@ -0,0 +1,26 @@
+{"current_steps": 2550, "total_steps": 2798, "loss": 0.9845, "lr": 1.9980979278401095e-05, "epoch": 0.22892539725289524, "percentage": 91.14, "elapsed_time": "0:00:56", "remaining_time": "0:00:05"}
+{"current_steps": 2560, "total_steps": 2798, "loss": 0.9732, "lr": 1.982924955463327e-05, "epoch": 0.22982314390878894, "percentage": 91.49, "elapsed_time": "0:02:25", "remaining_time": "0:00:13"}
+{"current_steps": 2570, "total_steps": 2798, "loss": 0.9493, "lr": 1.9528108298328235e-05, "epoch": 0.23072089056468265, "percentage": 91.85, "elapsed_time": "0:03:54", "remaining_time": "0:00:20"}
+{"current_steps": 2580, "total_steps": 2798, "loss": 0.9959, "lr": 1.9082156473635994e-05, "epoch": 0.23161863722057635, "percentage": 92.21, "elapsed_time": "0:05:23", "remaining_time": "0:00:27"}
+{"current_steps": 2590, "total_steps": 2798, "loss": 0.9221, "lr": 1.8498207522139434e-05, "epoch": 0.23251638387647006, "percentage": 92.57, "elapsed_time": "0:06:52", "remaining_time": "0:00:33"}
+{"current_steps": 2600, "total_steps": 2798, "loss": 0.9243, "lr": 1.7785183264196977e-05, "epoch": 0.23341413053236376, "percentage": 92.92, "elapsed_time": "0:08:21", "remaining_time": "0:00:38"}
+{"current_steps": 2610, "total_steps": 2798, "loss": 0.9099, "lr": 1.6953977587577846e-05, "epoch": 0.23431187718825747, "percentage": 93.28, "elapsed_time": "0:09:50", "remaining_time": "0:00:42"}
+{"current_steps": 2620, "total_steps": 2798, "loss": 0.9239, "lr": 1.601729000601297e-05, "epoch": 0.23520962384415117, "percentage": 93.64, "elapsed_time": "0:11:19", "remaining_time": "0:00:46"}
+{"current_steps": 2630, "total_steps": 2798, "loss": 0.8956, "lr": 1.4989431630624763e-05, "epoch": 0.23610737050004488, "percentage": 94.0, "elapsed_time": "0:12:48", "remaining_time": "0:00:49"}
+{"current_steps": 2640, "total_steps": 2798, "loss": 0.9649, "lr": 1.3886106518686746e-05, "epoch": 0.23700511715593858, "percentage": 94.35, "elapsed_time": "0:14:17", "remaining_time": "0:00:51"}
+{"current_steps": 2650, "total_steps": 2798, "loss": 0.8911, "lr": 1.2724171740359612e-05, "epoch": 0.2379028638118323, "percentage": 94.71, "elapsed_time": "0:15:46", "remaining_time": "0:00:52"}
+{"current_steps": 2660, "total_steps": 2798, "loss": 0.8959, "lr": 1.1521379829206218e-05, "epoch": 0.23880061046772602, "percentage": 95.07, "elapsed_time": "0:17:15", "remaining_time": "0:00:53"}
+{"current_steps": 2670, "total_steps": 2798, "loss": 0.8549, "lr": 1.029610755143595e-05, "epoch": 0.23969835712361973, "percentage": 95.43, "elapsed_time": "0:18:44", "remaining_time": "0:00:53"}
+{"current_steps": 2680, "total_steps": 2798, "loss": 0.9061, "lr": 9.067075137857348e-06, "epoch": 0.24059610377951343, "percentage": 95.78, "elapsed_time": "0:20:13", "remaining_time": "0:00:53"}
+{"current_steps": 2690, "total_steps": 2798, "loss": 0.9861, "lr": 7.853060268232525e-06, "epoch": 0.24149385043540714, "percentage": 96.14, "elapsed_time": "0:21:42", "remaining_time": "0:00:52"}
+{"current_steps": 2700, "total_steps": 2798, "loss": 0.9101, "lr": 6.672611177902054e-06, "epoch": 0.24239159709130084, "percentage": 96.5, "elapsed_time": "0:23:11", "remaining_time": "0:00:50"}
+{"current_steps": 2710, "total_steps": 2798, "loss": 0.9696, "lr": 5.543763269959312e-06, "epoch": 0.24328934374719455, "percentage": 96.85, "elapsed_time": "0:24:56", "remaining_time": "0:00:48"}
+{"current_steps": 2720, "total_steps": 2798, "loss": 0.9557, "lr": 4.483763562693867e-06, "epoch": 0.24418709040308825, "percentage": 97.21, "elapsed_time": "0:26:25", "remaining_time": "0:00:45"}
+{"current_steps": 2730, "total_steps": 2798, "loss": 0.8969, "lr": 3.508807182313187e-06, "epoch": 0.24508483705898196, "percentage": 97.57, "elapsed_time": "0:27:54", "remaining_time": "0:00:41"}
+{"current_steps": 2740, "total_steps": 2798, "loss": 0.8998, "lr": 2.6337899269187294e-06, "epoch": 0.24598258371487566, "percentage": 97.93, "elapsed_time": "0:29:23", "remaining_time": "0:00:37"}
+{"current_steps": 2750, "total_steps": 2798, "loss": 0.9262, "lr": 1.8720806821693126e-06, "epoch": 0.24688033037076937, "percentage": 98.28, "elapsed_time": "0:30:52", "remaining_time": "0:00:32"}
+{"current_steps": 2760, "total_steps": 2798, "loss": 0.9153, "lr": 1.2353171657619887e-06, "epoch": 0.24777807702666307, "percentage": 98.64, "elapsed_time": "0:32:21", "remaining_time": "0:00:26"}
+{"current_steps": 2770, "total_steps": 2798, "loss": 0.8894, "lr": 7.332281214330356e-07, "epoch": 0.24867582368255678, "percentage": 99.0, "elapsed_time": "0:33:50", "remaining_time": "0:00:20"}
+{"current_steps": 2780, "total_steps": 2798, "loss": 0.896, "lr": 3.73484679074808e-07, "epoch": 0.24957357033845048, "percentage": 99.36, "elapsed_time": "0:35:19", "remaining_time": "0:00:13"}
+{"current_steps": 2790, "total_steps": 2798, "loss": 0.8731, "lr": 1.61583151951702e-07, "epoch": 0.2504713169943442, "percentage": 99.71, "elapsed_time": "0:36:48", "remaining_time": "0:00:06"}
+{"current_steps": 2798, "total_steps": 2798, "epoch": 0.25118951431905917, "percentage": 100.0, "elapsed_time": "0:38:16", "remaining_time": "0:00:00"}

g1_0_baseline_original_code/250M/trainer_state.json

@@ -0,0 +1,1996 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.25118951431905917,
+  "eval_steps": 500,
+  "global_step": 2798,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008977466558937068,
+      "grad_norm": 7.634389400482178,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.9314,
+      "step": 10
+    },
+    {
+      "epoch": 0.0017954933117874136,
+      "grad_norm": 3.166454553604126,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 1.8945,
+      "step": 20
+    },
+    {
+      "epoch": 0.00269323996768112,
+      "grad_norm": 1.6780861616134644,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.6996,
+      "step": 30
+    },
+    {
+      "epoch": 0.003590986623574827,
+      "grad_norm": 1.238744854927063,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.5209,
+      "step": 40
+    },
+    {
+      "epoch": 0.004488733279468534,
+      "grad_norm": 0.8214995861053467,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 1.3707,
+      "step": 50
+    },
+    {
+      "epoch": 0.00538647993536224,
+      "grad_norm": 0.6225201487541199,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 1.2258,
+      "step": 60
+    },
+    {
+      "epoch": 0.006284226591255947,
+      "grad_norm": 0.6041092276573181,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 1.1689,
+      "step": 70
+    },
+    {
+      "epoch": 0.007181973247149654,
+      "grad_norm": 0.5930420756340027,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 1.1735,
+      "step": 80
+    },
+    {
+      "epoch": 0.008079719903043361,
+      "grad_norm": 0.603300929069519,
+      "learning_rate": 1.005e-05,
+      "loss": 1.1347,
+      "step": 90
+    },
+    {
+      "epoch": 0.008977466558937068,
+      "grad_norm": 0.5618454217910767,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 1.1165,
+      "step": 100
+    },
+    {
+      "epoch": 0.009875213214830775,
+      "grad_norm": 0.5842673182487488,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 1.0947,
+      "step": 110
+    },
+    {
+      "epoch": 0.01077295987072448,
+      "grad_norm": 0.5741180777549744,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 1.1028,
+      "step": 120
+    },
+    {
+      "epoch": 0.011670706526618188,
+      "grad_norm": 0.5997037291526794,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 1.1276,
+      "step": 130
+    },
+    {
+      "epoch": 0.012568453182511895,
+      "grad_norm": 0.5892444849014282,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 1.1735,
+      "step": 140
+    },
+    {
+      "epoch": 0.013466199838405602,
+      "grad_norm": 0.5162788033485413,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 1.0822,
+      "step": 150
+    },
+    {
+      "epoch": 0.014363946494299309,
+      "grad_norm": 0.6133973598480225,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 1.2064,
+      "step": 160
+    },
+    {
+      "epoch": 0.015261693150193016,
+      "grad_norm": 0.5716113448143005,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 1.1207,
+      "step": 170
+    },
+    {
+      "epoch": 0.016159439806086723,
+      "grad_norm": 0.8888991475105286,
+      "learning_rate": 2e-05,
+      "loss": 1.1478,
+      "step": 180
+    },
+    {
+      "epoch": 0.017057186461980428,
+      "grad_norm": 0.558070182800293,
+      "learning_rate": 2e-05,
+      "loss": 1.099,
+      "step": 190
+    },
+    {
+      "epoch": 0.017954933117874137,
+      "grad_norm": 0.6470697522163391,
+      "learning_rate": 2e-05,
+      "loss": 1.0708,
+      "step": 200
+    },
+    {
+      "epoch": 0.018852679773767842,
+      "grad_norm": 0.5357931852340698,
+      "learning_rate": 2e-05,
+      "loss": 1.1295,
+      "step": 210
+    },
+    {
+      "epoch": 0.01975042642966155,
+      "grad_norm": 0.5657357573509216,
+      "learning_rate": 2e-05,
+      "loss": 1.1604,
+      "step": 220
+    },
+    {
+      "epoch": 0.020648173085555256,
+      "grad_norm": 0.6352828741073608,
+      "learning_rate": 2e-05,
+      "loss": 1.119,
+      "step": 230
+    },
+    {
+      "epoch": 0.02154591974144896,
+      "grad_norm": 0.5701799988746643,
+      "learning_rate": 2e-05,
+      "loss": 1.0867,
+      "step": 240
+    },
+    {
+      "epoch": 0.02244366639734267,
+      "grad_norm": 0.54180508852005,
+      "learning_rate": 2e-05,
+      "loss": 1.0811,
+      "step": 250
+    },
+    {
+      "epoch": 0.023341413053236375,
+      "grad_norm": 0.5608647465705872,
+      "learning_rate": 2e-05,
+      "loss": 1.0937,
+      "step": 260
+    },
+    {
+      "epoch": 0.024239159709130084,
+      "grad_norm": 0.5483505129814148,
+      "learning_rate": 2e-05,
+      "loss": 1.0812,
+      "step": 270
+    },
+    {
+      "epoch": 0.02513690636502379,
+      "grad_norm": 0.5331147909164429,
+      "learning_rate": 2e-05,
+      "loss": 1.0576,
+      "step": 280
+    },
+    {
+      "epoch": 0.026034653020917498,
+      "grad_norm": 0.8958403468132019,
+      "learning_rate": 2e-05,
+      "loss": 1.1206,
+      "step": 290
+    },
+    {
+      "epoch": 0.026932399676811204,
+      "grad_norm": 0.5771209001541138,
+      "learning_rate": 2e-05,
+      "loss": 1.14,
+      "step": 300
+    },
+    {
+      "epoch": 0.027830146332704912,
+      "grad_norm": 0.508565366268158,
+      "learning_rate": 2e-05,
+      "loss": 1.0731,
+      "step": 310
+    },
+    {
+      "epoch": 0.028727892988598618,
+      "grad_norm": 0.6553564667701721,
+      "learning_rate": 2e-05,
+      "loss": 1.1244,
+      "step": 320
+    },
+    {
+      "epoch": 0.029625639644492323,
+      "grad_norm": 0.5307732820510864,
+      "learning_rate": 2e-05,
+      "loss": 1.109,
+      "step": 330
+    },
+    {
+      "epoch": 0.03052338630038603,
+      "grad_norm": 0.5320704579353333,
+      "learning_rate": 2e-05,
+      "loss": 1.0395,
+      "step": 340
+    },
+    {
+      "epoch": 0.03142113295627974,
+      "grad_norm": 0.48801037669181824,
+      "learning_rate": 2e-05,
+      "loss": 1.0765,
+      "step": 350
+    },
+    {
+      "epoch": 0.032318879612173446,
+      "grad_norm": 0.5760318636894226,
+      "learning_rate": 2e-05,
+      "loss": 1.0601,
+      "step": 360
+    },
+    {
+      "epoch": 0.03321662626806715,
+      "grad_norm": 0.5792495608329773,
+      "learning_rate": 2e-05,
+      "loss": 1.1067,
+      "step": 370
+    },
+    {
+      "epoch": 0.034114372923960856,
+      "grad_norm": 0.604045033454895,
+      "learning_rate": 2e-05,
+      "loss": 1.0792,
+      "step": 380
+    },
+    {
+      "epoch": 0.03501211957985457,
+      "grad_norm": 0.5748013854026794,
+      "learning_rate": 2e-05,
+      "loss": 1.1062,
+      "step": 390
+    },
+    {
+      "epoch": 0.035909866235748274,
+      "grad_norm": 0.5119761228561401,
+      "learning_rate": 2e-05,
+      "loss": 1.0289,
+      "step": 400
+    },
+    {
+      "epoch": 0.03680761289164198,
+      "grad_norm": 0.5493387579917908,
+      "learning_rate": 2e-05,
+      "loss": 1.0338,
+      "step": 410
+    },
+    {
+      "epoch": 0.037705359547535684,
+      "grad_norm": 0.6864805221557617,
+      "learning_rate": 2e-05,
+      "loss": 1.1149,
+      "step": 420
+    },
+    {
+      "epoch": 0.03860310620342939,
+      "grad_norm": 0.5241488218307495,
+      "learning_rate": 2e-05,
+      "loss": 1.0708,
+      "step": 430
+    },
+    {
+      "epoch": 0.0395008528593231,
+      "grad_norm": 0.5357916355133057,
+      "learning_rate": 2e-05,
+      "loss": 1.0752,
+      "step": 440
+    },
+    {
+      "epoch": 0.04039859951521681,
+      "grad_norm": 0.9240272641181946,
+      "learning_rate": 2e-05,
+      "loss": 1.0632,
+      "step": 450
+    },
+    {
+      "epoch": 0.04129634617111051,
+      "grad_norm": 0.5168442726135254,
+      "learning_rate": 2e-05,
+      "loss": 1.0794,
+      "step": 460
+    },
+    {
+      "epoch": 0.04219409282700422,
+      "grad_norm": 0.5578870177268982,
+      "learning_rate": 2e-05,
+      "loss": 1.0579,
+      "step": 470
+    },
+    {
+      "epoch": 0.04309183948289792,
+      "grad_norm": 4.224264144897461,
+      "learning_rate": 2e-05,
+      "loss": 1.1145,
+      "step": 480
+    },
+    {
+      "epoch": 0.043989586138791635,
+      "grad_norm": 0.5236369371414185,
+      "learning_rate": 2e-05,
+      "loss": 1.0754,
+      "step": 490
+    },
+    {
+      "epoch": 0.04488733279468534,
+      "grad_norm": 0.5369593501091003,
+      "learning_rate": 2e-05,
+      "loss": 1.0746,
+      "step": 500
+    },
+    {
+      "epoch": 0.045785079450579046,
+      "grad_norm": 0.5629836916923523,
+      "learning_rate": 2e-05,
+      "loss": 1.0203,
+      "step": 510
+    },
+    {
+      "epoch": 0.04668282610647275,
+      "grad_norm": 1.1909027099609375,
+      "learning_rate": 2e-05,
+      "loss": 1.0484,
+      "step": 520
+    },
+    {
+      "epoch": 0.04758057276236646,
+      "grad_norm": 0.5194602608680725,
+      "learning_rate": 2e-05,
+      "loss": 1.1053,
+      "step": 530
+    },
+    {
+      "epoch": 0.04847831941826017,
+      "grad_norm": 0.49282965064048767,
+      "learning_rate": 2e-05,
+      "loss": 1.0526,
+      "step": 540
+    },
+    {
+      "epoch": 0.049376066074153874,
+      "grad_norm": 0.5095282196998596,
+      "learning_rate": 2e-05,
+      "loss": 1.0829,
+      "step": 550
+    },
+    {
+      "epoch": 0.05027381273004758,
+      "grad_norm": 0.574981153011322,
+      "learning_rate": 2e-05,
+      "loss": 1.0399,
+      "step": 560
+    },
+    {
+      "epoch": 0.051171559385941284,
+      "grad_norm": 0.5354433655738831,
+      "learning_rate": 2e-05,
+      "loss": 1.0195,
+      "step": 570
+    },
+    {
+      "epoch": 0.052069306041834997,
+      "grad_norm": 0.5005495548248291,
+      "learning_rate": 2e-05,
+      "loss": 1.0515,
+      "step": 580
+    },
+    {
+      "epoch": 0.0529670526977287,
+      "grad_norm": 0.4863434135913849,
+      "learning_rate": 2e-05,
+      "loss": 1.0854,
+      "step": 590
+    },
+    {
+      "epoch": 0.05386479935362241,
+      "grad_norm": 0.8269434571266174,
+      "learning_rate": 2e-05,
+      "loss": 1.0621,
+      "step": 600
+    },
+    {
+      "epoch": 0.05476254600951611,
+      "grad_norm": 0.5037260055541992,
+      "learning_rate": 2e-05,
+      "loss": 1.0329,
+      "step": 610
+    },
+    {
+      "epoch": 0.055660292665409825,
+      "grad_norm": 0.49951377511024475,
+      "learning_rate": 2e-05,
+      "loss": 1.0823,
+      "step": 620
+    },
+    {
+      "epoch": 0.05655803932130353,
+      "grad_norm": 0.5041629076004028,
+      "learning_rate": 2e-05,
+      "loss": 1.0681,
+      "step": 630
+    },
+    {
+      "epoch": 0.057455785977197235,
+      "grad_norm": 0.4688110947608948,
+      "learning_rate": 2e-05,
+      "loss": 1.0772,
+      "step": 640
+    },
+    {
+      "epoch": 0.05835353263309094,
+      "grad_norm": 0.5006560683250427,
+      "learning_rate": 2e-05,
+      "loss": 1.0695,
+      "step": 650
+    },
+    {
+      "epoch": 0.059251279288984646,
+      "grad_norm": 0.5676978230476379,
+      "learning_rate": 2e-05,
+      "loss": 1.0395,
+      "step": 660
+    },
+    {
+      "epoch": 0.06014902594487836,
+      "grad_norm": 1.2871171236038208,
+      "learning_rate": 2e-05,
+      "loss": 1.0843,
+      "step": 670
+    },
+    {
+      "epoch": 0.06104677260077206,
+      "grad_norm": 0.701192319393158,
+      "learning_rate": 2e-05,
+      "loss": 1.0473,
+      "step": 680
+    },
+    {
+      "epoch": 0.06194451925666577,
+      "grad_norm": 0.7122907042503357,
+      "learning_rate": 2e-05,
+      "loss": 1.0651,
+      "step": 690
+    },
+    {
+      "epoch": 0.06284226591255948,
+      "grad_norm": 0.47654181718826294,
+      "learning_rate": 2e-05,
+      "loss": 1.0682,
+      "step": 700
+    },
+    {
+      "epoch": 0.06374001256845319,
+      "grad_norm": 0.513289213180542,
+      "learning_rate": 2e-05,
+      "loss": 1.0388,
+      "step": 710
+    },
+    {
+      "epoch": 0.06463775922434689,
+      "grad_norm": 0.6258441209793091,
+      "learning_rate": 2e-05,
+      "loss": 1.0492,
+      "step": 720
+    },
+    {
+      "epoch": 0.0655355058802406,
+      "grad_norm": 0.5013813376426697,
+      "learning_rate": 2e-05,
+      "loss": 1.0458,
+      "step": 730
+    },
+    {
+      "epoch": 0.0664332525361343,
+      "grad_norm": 0.5458043217658997,
+      "learning_rate": 2e-05,
+      "loss": 1.0525,
+      "step": 740
+    },
+    {
+      "epoch": 0.06733099919202801,
+      "grad_norm": 0.5081648230552673,
+      "learning_rate": 2e-05,
+      "loss": 1.0201,
+      "step": 750
+    },
+    {
+      "epoch": 0.06822874584792171,
+      "grad_norm": 0.49785295128822327,
+      "learning_rate": 2e-05,
+      "loss": 1.1267,
+      "step": 760
+    },
+    {
+      "epoch": 0.06912649250381542,
+      "grad_norm": 0.5245663523674011,
+      "learning_rate": 2e-05,
+      "loss": 1.0173,
+      "step": 770
+    },
+    {
+      "epoch": 0.07002423915970914,
+      "grad_norm": 0.48743847012519836,
+      "learning_rate": 2e-05,
+      "loss": 1.0074,
+      "step": 780
+    },
+    {
+      "epoch": 0.07092198581560284,
+      "grad_norm": 0.6617560982704163,
+      "learning_rate": 2e-05,
+      "loss": 1.044,
+      "step": 790
+    },
+    {
+      "epoch": 0.07181973247149655,
+      "grad_norm": 0.5401202440261841,
+      "learning_rate": 2e-05,
+      "loss": 1.0229,
+      "step": 800
+    },
+    {
+      "epoch": 0.07271747912739025,
+      "grad_norm": 0.543354868888855,
+      "learning_rate": 2e-05,
+      "loss": 1.0567,
+      "step": 810
+    },
+    {
+      "epoch": 0.07361522578328396,
+      "grad_norm": 1.1279566287994385,
+      "learning_rate": 2e-05,
+      "loss": 1.0055,
+      "step": 820
+    },
+    {
+      "epoch": 0.07451297243917766,
+      "grad_norm": 0.5519621968269348,
+      "learning_rate": 2e-05,
+      "loss": 1.0213,
+      "step": 830
+    },
+    {
+      "epoch": 0.07541071909507137,
+      "grad_norm": 1.1007333993911743,
+      "learning_rate": 2e-05,
+      "loss": 1.0484,
+      "step": 840
+    },
+    {
+      "epoch": 0.07630846575096507,
+      "grad_norm": 0.47256144881248474,
+      "learning_rate": 2e-05,
+      "loss": 1.0677,
+      "step": 850
+    },
+    {
+      "epoch": 0.07720621240685878,
+      "grad_norm": 0.4924080967903137,
+      "learning_rate": 2e-05,
+      "loss": 1.0573,
+      "step": 860
+    },
+    {
+      "epoch": 0.07810395906275248,
+      "grad_norm": 0.5096149444580078,
+      "learning_rate": 2e-05,
+      "loss": 1.1057,
+      "step": 870
+    },
+    {
+      "epoch": 0.0790017057186462,
+      "grad_norm": 0.5318610072135925,
+      "learning_rate": 2e-05,
+      "loss": 1.0146,
+      "step": 880
+    },
+    {
+      "epoch": 0.07989945237453991,
+      "grad_norm": 0.675080418586731,
+      "learning_rate": 2e-05,
+      "loss": 1.0599,
+      "step": 890
+    },
+    {
+      "epoch": 0.08079719903043361,
+      "grad_norm": 0.47846636176109314,
+      "learning_rate": 2e-05,
+      "loss": 1.0417,
+      "step": 900
+    },
+    {
+      "epoch": 0.08169494568632732,
+      "grad_norm": 0.4796086847782135,
+      "learning_rate": 2e-05,
+      "loss": 1.0107,
+      "step": 910
+    },
+    {
+      "epoch": 0.08259269234222102,
+      "grad_norm": 0.47223231196403503,
+      "learning_rate": 2e-05,
+      "loss": 1.0659,
+      "step": 920
+    },
+    {
+      "epoch": 0.08349043899811473,
+      "grad_norm": 0.5378100275993347,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 930
+    },
+    {
+      "epoch": 0.08438818565400844,
+      "grad_norm": 0.5030729174613953,
+      "learning_rate": 2e-05,
+      "loss": 1.0183,
+      "step": 940
+    },
+    {
+      "epoch": 0.08528593230990214,
+      "grad_norm": 0.5262132287025452,
+      "learning_rate": 2e-05,
+      "loss": 1.0381,
+      "step": 950
+    },
+    {
+      "epoch": 0.08618367896579585,
+      "grad_norm": 0.5390843749046326,
+      "learning_rate": 2e-05,
+      "loss": 1.01,
+      "step": 960
+    },
+    {
+      "epoch": 0.08708142562168956,
+      "grad_norm": 0.5111972689628601,
+      "learning_rate": 2e-05,
+      "loss": 1.0757,
+      "step": 970
+    },
+    {
+      "epoch": 0.08797917227758327,
+      "grad_norm": 0.5065791010856628,
+      "learning_rate": 2e-05,
+      "loss": 1.0301,
+      "step": 980
+    },
+    {
+      "epoch": 0.08887691893347698,
+      "grad_norm": 0.5182551741600037,
+      "learning_rate": 2e-05,
+      "loss": 1.0442,
+      "step": 990
+    },
+    {
+      "epoch": 0.08977466558937068,
+      "grad_norm": 0.5045287609100342,
+      "learning_rate": 2e-05,
+      "loss": 1.0133,
+      "step": 1000
+    },
+    {
+      "epoch": 0.09067241224526439,
+      "grad_norm": 0.47217923402786255,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 1010
+    },
+    {
+      "epoch": 0.09157015890115809,
+      "grad_norm": 0.6045490503311157,
+      "learning_rate": 2e-05,
+      "loss": 1.0759,
+      "step": 1020
+    },
+    {
+      "epoch": 0.0924679055570518,
+      "grad_norm": 0.49416762590408325,
+      "learning_rate": 2e-05,
+      "loss": 0.9857,
+      "step": 1030
+    },
+    {
+      "epoch": 0.0933656522129455,
+      "grad_norm": 0.9547766447067261,
+      "learning_rate": 2e-05,
+      "loss": 1.1359,
+      "step": 1040
+    },
+    {
+      "epoch": 0.09426339886883921,
+      "grad_norm": 0.5142229795455933,
+      "learning_rate": 2e-05,
+      "loss": 1.0329,
+      "step": 1050
+    },
+    {
+      "epoch": 0.09516114552473293,
+      "grad_norm": 0.5049414038658142,
+      "learning_rate": 2e-05,
+      "loss": 0.9621,
+      "step": 1060
+    },
+    {
+      "epoch": 0.09605889218062663,
+      "grad_norm": 0.5428811311721802,
+      "learning_rate": 2e-05,
+      "loss": 1.0414,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09695663883652034,
+      "grad_norm": 0.5135433077812195,
+      "learning_rate": 2e-05,
+      "loss": 1.0206,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09785438549241404,
+      "grad_norm": 0.47476571798324585,
+      "learning_rate": 2e-05,
+      "loss": 1.0308,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09875213214830775,
+      "grad_norm": 0.748688280582428,
+      "learning_rate": 2e-05,
+      "loss": 1.0606,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09964987880420145,
+      "grad_norm": 0.45084625482559204,
+      "learning_rate": 2e-05,
+      "loss": 1.0314,
+      "step": 1110
+    },
+    {
+      "epoch": 0.10054762546009516,
+      "grad_norm": 0.5020058155059814,
+      "learning_rate": 2e-05,
+      "loss": 1.0001,
+      "step": 1120
+    },
+    {
+      "epoch": 0.10144537211598886,
+      "grad_norm": 0.7483564019203186,
+      "learning_rate": 2e-05,
+      "loss": 1.0236,
+      "step": 1130
+    },
+    {
+      "epoch": 0.10234311877188257,
+      "grad_norm": 0.5015250444412231,
+      "learning_rate": 2e-05,
+      "loss": 1.0947,
+      "step": 1140
+    },
+    {
+      "epoch": 0.10324086542777629,
+      "grad_norm": 0.4564529061317444,
+      "learning_rate": 2e-05,
+      "loss": 1.0038,
+      "step": 1150
+    },
+    {
+      "epoch": 0.10413861208366999,
+      "grad_norm": 0.47453996539115906,
+      "learning_rate": 2e-05,
+      "loss": 0.9863,
+      "step": 1160
+    },
+    {
+      "epoch": 0.1050363587395637,
+      "grad_norm": 0.4623263478279114,
+      "learning_rate": 2e-05,
+      "loss": 1.0008,
+      "step": 1170
+    },
+    {
+      "epoch": 0.1059341053954574,
+      "grad_norm": 0.4754187762737274,
+      "learning_rate": 2e-05,
+      "loss": 0.9987,
+      "step": 1180
+    },
+    {
+      "epoch": 0.10683185205135111,
+      "grad_norm": 0.4644615650177002,
+      "learning_rate": 2e-05,
+      "loss": 0.9815,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10772959870724481,
+      "grad_norm": 0.482634037733078,
+      "learning_rate": 2e-05,
+      "loss": 0.9772,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10862734536313852,
+      "grad_norm": 0.5518064498901367,
+      "learning_rate": 2e-05,
+      "loss": 1.0926,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10952509201903222,
+      "grad_norm": 0.5161309242248535,
+      "learning_rate": 2e-05,
+      "loss": 1.0023,
+      "step": 1220
+    },
+    {
+      "epoch": 0.11042283867492593,
+      "grad_norm": 0.47377634048461914,
+      "learning_rate": 2e-05,
+      "loss": 1.0061,
+      "step": 1230
+    },
+    {
+      "epoch": 0.11132058533081965,
+      "grad_norm": 0.5026335716247559,
+      "learning_rate": 2e-05,
+      "loss": 0.9962,
+      "step": 1240
+    },
+    {
+      "epoch": 0.11221833198671335,
+      "grad_norm": 0.5823997855186462,
+      "learning_rate": 2e-05,
+      "loss": 1.0312,
+      "step": 1250
+    },
+    {
+      "epoch": 0.11311607864260706,
+      "grad_norm": 0.44196802377700806,
+      "learning_rate": 2e-05,
+      "loss": 1.0227,
+      "step": 1260
+    },
+    {
+      "epoch": 0.11401382529850076,
+      "grad_norm": 0.48227500915527344,
+      "learning_rate": 2e-05,
+      "loss": 1.041,
+      "step": 1270
+    },
+    {
+      "epoch": 0.11491157195439447,
+      "grad_norm": 0.48953211307525635,
+      "learning_rate": 2e-05,
+      "loss": 1.0208,
+      "step": 1280
+    },
+    {
+      "epoch": 0.11580931861028818,
+      "grad_norm": 0.4923247992992401,
+      "learning_rate": 2e-05,
+      "loss": 1.0013,
+      "step": 1290
+    },
+    {
+      "epoch": 0.11670706526618188,
+      "grad_norm": 0.4701191186904907,
+      "learning_rate": 2e-05,
+      "loss": 1.0063,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11760481192207559,
+      "grad_norm": 0.5244788527488708,
+      "learning_rate": 2e-05,
+      "loss": 0.9743,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11850255857796929,
+      "grad_norm": 0.5617148280143738,
+      "learning_rate": 2e-05,
+      "loss": 1.1172,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11940030523386301,
+      "grad_norm": 0.49794480204582214,
+      "learning_rate": 2e-05,
+      "loss": 0.9922,
+      "step": 1330
+    },
+    {
+      "epoch": 0.12029805188975672,
+      "grad_norm": 0.48185423016548157,
+      "learning_rate": 2e-05,
+      "loss": 0.966,
+      "step": 1340
+    },
+    {
+      "epoch": 0.12119579854565042,
+      "grad_norm": 0.4720144271850586,
+      "learning_rate": 2e-05,
+      "loss": 0.9698,
+      "step": 1350
+    },
+    {
+      "epoch": 0.12209354520154413,
+      "grad_norm": 0.4971640408039093,
+      "learning_rate": 2e-05,
+      "loss": 0.9506,
+      "step": 1360
+    },
+    {
+      "epoch": 0.12299129185743783,
+      "grad_norm": 0.582372784614563,
+      "learning_rate": 2e-05,
+      "loss": 0.9695,
+      "step": 1370
+    },
+    {
+      "epoch": 0.12388903851333154,
+      "grad_norm": 0.5824956297874451,
+      "learning_rate": 2e-05,
+      "loss": 0.9911,
+      "step": 1380
+    },
+    {
+      "epoch": 0.12478678516922524,
+      "grad_norm": 0.49514421820640564,
+      "learning_rate": 2e-05,
+      "loss": 0.9874,
+      "step": 1390
+    },
+    {
+      "epoch": 0.12568453182511896,
+      "grad_norm": 0.520560622215271,
+      "learning_rate": 2e-05,
+      "loss": 1.0279,
+      "step": 1400
+    },
+    {
+      "epoch": 0.12658227848101267,
+      "grad_norm": 0.5677764415740967,
+      "learning_rate": 2e-05,
+      "loss": 1.0537,
+      "step": 1410
+    },
+    {
+      "epoch": 0.12748002513690637,
+      "grad_norm": 0.5025660991668701,
+      "learning_rate": 2e-05,
+      "loss": 0.978,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12837777179280008,
+      "grad_norm": 0.4592083990573883,
+      "learning_rate": 2e-05,
+      "loss": 0.9955,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12927551844869378,
+      "grad_norm": 0.4699345529079437,
+      "learning_rate": 2e-05,
+      "loss": 0.9694,
+      "step": 1440
+    },
+    {
+      "epoch": 0.1301732651045875,
+      "grad_norm": 0.48181068897247314,
+      "learning_rate": 2e-05,
+      "loss": 0.9672,
+      "step": 1450
+    },
+    {
+      "epoch": 0.1310710117604812,
+      "grad_norm": 2.121140956878662,
+      "learning_rate": 2e-05,
+      "loss": 0.9885,
+      "step": 1460
+    },
+    {
+      "epoch": 0.1319687584163749,
+      "grad_norm": 0.47388073801994324,
+      "learning_rate": 2e-05,
+      "loss": 1.0243,
+      "step": 1470
+    },
+    {
+      "epoch": 0.1328665050722686,
+      "grad_norm": 0.6105418801307678,
+      "learning_rate": 2e-05,
+      "loss": 1.024,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1337642517281623,
+      "grad_norm": 0.5576895475387573,
+      "learning_rate": 2e-05,
+      "loss": 0.9856,
+      "step": 1490
+    },
+    {
+      "epoch": 0.13466199838405601,
+      "grad_norm": 0.43749862909317017,
+      "learning_rate": 2e-05,
+      "loss": 0.9399,
+      "step": 1500
+    },
+    {
+      "epoch": 0.13555974503994972,
+      "grad_norm": 0.5869466066360474,
+      "learning_rate": 2e-05,
+      "loss": 1.0006,
+      "step": 1510
+    },
+    {
+      "epoch": 0.13645749169584342,
+      "grad_norm": 0.5816555023193359,
+      "learning_rate": 2e-05,
+      "loss": 0.9505,
+      "step": 1520
+    },
+    {
+      "epoch": 0.13735523835173713,
+      "grad_norm": 0.4605148732662201,
+      "learning_rate": 2e-05,
+      "loss": 1.0417,
+      "step": 1530
+    },
+    {
+      "epoch": 0.13825298500763084,
+      "grad_norm": 0.5583087801933289,
+      "learning_rate": 2e-05,
+      "loss": 1.0041,
+      "step": 1540
+    },
+    {
+      "epoch": 0.13915073166352454,
+      "grad_norm": 0.48093125224113464,
+      "learning_rate": 2e-05,
+      "loss": 0.9559,
+      "step": 1550
+    },
+    {
+      "epoch": 0.14004847831941827,
+      "grad_norm": 0.5225452184677124,
+      "learning_rate": 2e-05,
+      "loss": 0.9674,
+      "step": 1560
+    },
+    {
+      "epoch": 0.14094622497531198,
+      "grad_norm": 0.4931219220161438,
+      "learning_rate": 2e-05,
+      "loss": 1.032,
+      "step": 1570
+    },
+    {
+      "epoch": 0.14184397163120568,
+      "grad_norm": 0.4754665791988373,
+      "learning_rate": 2e-05,
+      "loss": 0.9832,
+      "step": 1580
+    },
+    {
+      "epoch": 0.1427417182870994,
+      "grad_norm": 0.5914980173110962,
+      "learning_rate": 2e-05,
+      "loss": 0.9563,
+      "step": 1590
+    },
+    {
+      "epoch": 0.1436394649429931,
+      "grad_norm": 0.4562107026576996,
+      "learning_rate": 2e-05,
+      "loss": 1.0162,
+      "step": 1600
+    },
+    {
+      "epoch": 0.1445372115988868,
+      "grad_norm": 0.4780389070510864,
+      "learning_rate": 2e-05,
+      "loss": 0.954,
+      "step": 1610
+    },
+    {
+      "epoch": 0.1454349582547805,
+      "grad_norm": 0.8750007748603821,
+      "learning_rate": 2e-05,
+      "loss": 0.9849,
+      "step": 1620
+    },
+    {
+      "epoch": 0.1463327049106742,
+      "grad_norm": 0.48428502678871155,
+      "learning_rate": 2e-05,
+      "loss": 1.0274,
+      "step": 1630
+    },
+    {
+      "epoch": 0.14723045156656792,
+      "grad_norm": 0.4305723309516907,
+      "learning_rate": 2e-05,
+      "loss": 0.9729,
+      "step": 1640
+    },
+    {
+      "epoch": 0.14812819822246162,
+      "grad_norm": 0.5047057867050171,
+      "learning_rate": 2e-05,
+      "loss": 0.997,
+      "step": 1650
+    },
+    {
+      "epoch": 0.14902594487835533,
+      "grad_norm": 0.5030443072319031,
+      "learning_rate": 2e-05,
+      "loss": 1.0052,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14992369153424903,
+      "grad_norm": 0.6058238744735718,
+      "learning_rate": 2e-05,
+      "loss": 0.9561,
+      "step": 1670
+    },
+    {
+      "epoch": 0.15082143819014274,
+      "grad_norm": 0.4600324034690857,
+      "learning_rate": 2e-05,
+      "loss": 0.9747,
+      "step": 1680
+    },
+    {
+      "epoch": 0.15171918484603644,
+      "grad_norm": 0.5196220278739929,
+      "learning_rate": 2e-05,
+      "loss": 0.9739,
+      "step": 1690
+    },
+    {
+      "epoch": 0.15261693150193015,
+      "grad_norm": 0.5066395401954651,
+      "learning_rate": 2e-05,
+      "loss": 0.9805,
+      "step": 1700
+    },
+    {
+      "epoch": 0.15351467815782385,
+      "grad_norm": 0.4423527121543884,
+      "learning_rate": 2e-05,
+      "loss": 1.0143,
+      "step": 1710
+    },
+    {
+      "epoch": 0.15441242481371756,
+      "grad_norm": 0.484889954328537,
+      "learning_rate": 2e-05,
+      "loss": 1.0116,
+      "step": 1720
+    },
+    {
+      "epoch": 0.15531017146961126,
+      "grad_norm": 0.4426444172859192,
+      "learning_rate": 2e-05,
+      "loss": 0.9874,
+      "step": 1730
+    },
+    {
+      "epoch": 0.15620791812550497,
+      "grad_norm": 0.48062387108802795,
+      "learning_rate": 2e-05,
+      "loss": 1.016,
+      "step": 1740
+    },
+    {
+      "epoch": 0.1571056647813987,
+      "grad_norm": 0.5096336603164673,
+      "learning_rate": 2e-05,
+      "loss": 0.9822,
+      "step": 1750
+    },
+    {
+      "epoch": 0.1580034114372924,
+      "grad_norm": 0.5110201835632324,
+      "learning_rate": 2e-05,
+      "loss": 0.9701,
+      "step": 1760
+    },
+    {
+      "epoch": 0.1589011580931861,
+      "grad_norm": 0.5047339200973511,
+      "learning_rate": 2e-05,
+      "loss": 1.0506,
+      "step": 1770
+    },
+    {
+      "epoch": 0.15979890474907982,
+      "grad_norm": 0.5041124820709229,
+      "learning_rate": 2e-05,
+      "loss": 0.9917,
+      "step": 1780
+    },
+    {
+      "epoch": 0.16069665140497352,
+      "grad_norm": 0.5139462947845459,
+      "learning_rate": 2e-05,
+      "loss": 1.0379,
+      "step": 1790
+    },
+    {
+      "epoch": 0.16159439806086723,
+      "grad_norm": 0.45225703716278076,
+      "learning_rate": 2e-05,
+      "loss": 0.994,
+      "step": 1800
+    },
+    {
+      "epoch": 0.16249214471676093,
+      "grad_norm": 0.48371419310569763,
+      "learning_rate": 2e-05,
+      "loss": 0.9789,
+      "step": 1810
+    },
+    {
+      "epoch": 0.16338989137265464,
+      "grad_norm": 1.0779194831848145,
+      "learning_rate": 2e-05,
+      "loss": 1.009,
+      "step": 1820
+    },
+    {
+      "epoch": 0.16428763802854834,
+      "grad_norm": 0.5898749232292175,
+      "learning_rate": 2e-05,
+      "loss": 1.0905,
+      "step": 1830
+    },
+    {
+      "epoch": 0.16518538468444205,
+      "grad_norm": 0.5248507857322693,
+      "learning_rate": 2e-05,
+      "loss": 0.9871,
+      "step": 1840
+    },
+    {
+      "epoch": 0.16608313134033575,
+      "grad_norm": 0.5213241577148438,
+      "learning_rate": 2e-05,
+      "loss": 1.0232,
+      "step": 1850
+    },
+    {
+      "epoch": 0.16698087799622946,
+      "grad_norm": 0.5059240460395813,
+      "learning_rate": 2e-05,
+      "loss": 0.941,
+      "step": 1860
+    },
+    {
+      "epoch": 0.16787862465212317,
+      "grad_norm": 0.49990788102149963,
+      "learning_rate": 2e-05,
+      "loss": 0.9661,
+      "step": 1870
+    },
+    {
+      "epoch": 0.16877637130801687,
+      "grad_norm": 0.5290054678916931,
+      "learning_rate": 2e-05,
+      "loss": 0.9759,
+      "step": 1880
+    },
+    {
+      "epoch": 0.16967411796391058,
+      "grad_norm": 0.48694655299186707,
+      "learning_rate": 2e-05,
+      "loss": 0.9813,
+      "step": 1890
+    },
+    {
+      "epoch": 0.17057186461980428,
+      "grad_norm": 0.7233108878135681,
+      "learning_rate": 2e-05,
+      "loss": 0.9737,
+      "step": 1900
+    },
+    {
+      "epoch": 0.171469611275698,
+      "grad_norm": 0.496572881937027,
+      "learning_rate": 2e-05,
+      "loss": 0.9429,
+      "step": 1910
+    },
+    {
+      "epoch": 0.1723673579315917,
+      "grad_norm": 0.4263172745704651,
+      "learning_rate": 2e-05,
+      "loss": 0.9352,
+      "step": 1920
+    },
+    {
+      "epoch": 0.17326510458748542,
+      "grad_norm": 0.4787362217903137,
+      "learning_rate": 2e-05,
+      "loss": 0.9401,
+      "step": 1930
+    },
+    {
+      "epoch": 0.17416285124337913,
+      "grad_norm": 0.46687498688697815,
+      "learning_rate": 2e-05,
+      "loss": 0.96,
+      "step": 1940
+    },
+    {
+      "epoch": 0.17506059789927284,
+      "grad_norm": 0.5431246757507324,
+      "learning_rate": 2e-05,
+      "loss": 0.9933,
+      "step": 1950
+    },
+    {
+      "epoch": 0.17595834455516654,
+      "grad_norm": 0.5195553302764893,
+      "learning_rate": 2e-05,
+      "loss": 0.9357,
+      "step": 1960
+    },
+    {
+      "epoch": 0.17685609121106025,
+      "grad_norm": 0.47831055521965027,
+      "learning_rate": 2e-05,
+      "loss": 0.9931,
+      "step": 1970
+    },
+    {
+      "epoch": 0.17775383786695395,
+      "grad_norm": 0.4682317078113556,
+      "learning_rate": 2e-05,
+      "loss": 0.926,
+      "step": 1980
+    },
+    {
+      "epoch": 0.17865158452284766,
+      "grad_norm": 0.5072707533836365,
+      "learning_rate": 2e-05,
+      "loss": 0.9792,
+      "step": 1990
+    },
+    {
+      "epoch": 0.17954933117874136,
+      "grad_norm": 0.5063421130180359,
+      "learning_rate": 2e-05,
+      "loss": 0.9729,
+      "step": 2000
+    },
+    {
+      "epoch": 0.18044707783463507,
+      "grad_norm": 0.47064459323883057,
+      "learning_rate": 2e-05,
+      "loss": 0.9511,
+      "step": 2010
+    },
+    {
+      "epoch": 0.18134482449052877,
+      "grad_norm": 0.48214080929756165,
+      "learning_rate": 2e-05,
+      "loss": 0.9992,
+      "step": 2020
+    },
+    {
+      "epoch": 0.18224257114642248,
+      "grad_norm": 0.48727652430534363,
+      "learning_rate": 2e-05,
+      "loss": 0.9414,
+      "step": 2030
+    },
+    {
+      "epoch": 0.18314031780231618,
+      "grad_norm": 0.5159448981285095,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2040
+    },
+    {
+      "epoch": 0.1840380644582099,
+      "grad_norm": 0.4612741768360138,
+      "learning_rate": 2e-05,
+      "loss": 0.9929,
+      "step": 2050
+    },
+    {
+      "epoch": 0.1849358111141036,
+      "grad_norm": 0.4544714689254761,
+      "learning_rate": 2e-05,
+      "loss": 0.9007,
+      "step": 2060
+    },
+    {
+      "epoch": 0.1858335577699973,
+      "grad_norm": 0.4296947419643402,
+      "learning_rate": 2e-05,
+      "loss": 0.9784,
+      "step": 2070
+    },
+    {
+      "epoch": 0.186731304425891,
+      "grad_norm": 0.455415815114975,
+      "learning_rate": 2e-05,
+      "loss": 0.9248,
+      "step": 2080
+    },
+    {
+      "epoch": 0.1876290510817847,
+      "grad_norm": 0.4618610441684723,
+      "learning_rate": 2e-05,
+      "loss": 0.9991,
+      "step": 2090
+    },
+    {
+      "epoch": 0.18852679773767841,
+      "grad_norm": 0.5599287152290344,
+      "learning_rate": 2e-05,
+      "loss": 0.9861,
+      "step": 2100
+    },
+    {
+      "epoch": 0.18942454439357215,
+      "grad_norm": 0.49464237689971924,
+      "learning_rate": 2e-05,
+      "loss": 1.0128,
+      "step": 2110
+    },
+    {
+      "epoch": 0.19032229104946585,
+      "grad_norm": 0.5259017944335938,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2120
+    },
+    {
+      "epoch": 0.19122003770535956,
+      "grad_norm": 0.4654732942581177,
+      "learning_rate": 2e-05,
+      "loss": 1.0427,
+      "step": 2130
+    },
+    {
+      "epoch": 0.19211778436125326,
+      "grad_norm": 0.7427297234535217,
+      "learning_rate": 2e-05,
+      "loss": 0.953,
+      "step": 2140
+    },
+    {
+      "epoch": 0.19301553101714697,
+      "grad_norm": 0.47707414627075195,
+      "learning_rate": 2e-05,
+      "loss": 0.956,
+      "step": 2150
+    },
+    {
+      "epoch": 0.19391327767304067,
+      "grad_norm": 0.4793659448623657,
+      "learning_rate": 2e-05,
+      "loss": 0.9417,
+      "step": 2160
+    },
+    {
+      "epoch": 0.19481102432893438,
+      "grad_norm": 0.5135785341262817,
+      "learning_rate": 2e-05,
+      "loss": 0.9752,
+      "step": 2170
+    },
+    {
+      "epoch": 0.19570877098482808,
+      "grad_norm": 0.5346530079841614,
+      "learning_rate": 2e-05,
+      "loss": 0.9438,
+      "step": 2180
+    },
+    {
+      "epoch": 0.1966065176407218,
+      "grad_norm": 0.4482640027999878,
+      "learning_rate": 2e-05,
+      "loss": 1.005,
+      "step": 2190
+    },
+    {
+      "epoch": 0.1975042642966155,
+      "grad_norm": 0.46668392419815063,
+      "learning_rate": 2e-05,
+      "loss": 0.9477,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1984020109525092,
+      "grad_norm": 0.46946898102760315,
+      "learning_rate": 2e-05,
+      "loss": 0.9564,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1992997576084029,
+      "grad_norm": 0.5270060896873474,
+      "learning_rate": 2e-05,
+      "loss": 0.9895,
+      "step": 2220
+    },
+    {
+      "epoch": 0.2001975042642966,
+      "grad_norm": 0.5055740475654602,
+      "learning_rate": 2e-05,
+      "loss": 0.8949,
+      "step": 2230
+    },
+    {
+      "epoch": 0.20109525092019032,
+      "grad_norm": 0.4616740047931671,
+      "learning_rate": 2e-05,
+      "loss": 0.97,
+      "step": 2240
+    },
+    {
+      "epoch": 0.20199299757608402,
+      "grad_norm": 0.4541808068752289,
+      "learning_rate": 2e-05,
+      "loss": 0.9442,
+      "step": 2250
+    },
+    {
+      "epoch": 0.20289074423197773,
+      "grad_norm": 0.5195493698120117,
+      "learning_rate": 2e-05,
+      "loss": 0.9948,
+      "step": 2260
+    },
+    {
+      "epoch": 0.20378849088787143,
+      "grad_norm": 0.5284701585769653,
+      "learning_rate": 2e-05,
+      "loss": 0.9796,
+      "step": 2270
+    },
+    {
+      "epoch": 0.20468623754376514,
+      "grad_norm": 0.47338250279426575,
+      "learning_rate": 2e-05,
+      "loss": 0.9378,
+      "step": 2280
+    },
+    {
+      "epoch": 0.20558398419965884,
+      "grad_norm": 0.4906748831272125,
+      "learning_rate": 2e-05,
+      "loss": 0.9891,
+      "step": 2290
+    },
+    {
+      "epoch": 0.20648173085555258,
+      "grad_norm": 0.4232115149497986,
+      "learning_rate": 2e-05,
+      "loss": 0.9159,
+      "step": 2300
+    },
+    {
+      "epoch": 0.20737947751144628,
+      "grad_norm": 0.4847649037837982,
+      "learning_rate": 2e-05,
+      "loss": 0.9513,
+      "step": 2310
+    },
+    {
+      "epoch": 0.20827722416733999,
+      "grad_norm": 0.5703267455101013,
+      "learning_rate": 2e-05,
+      "loss": 0.9478,
+      "step": 2320
+    },
+    {
+      "epoch": 0.2091749708232337,
+      "grad_norm": 0.48311203718185425,
+      "learning_rate": 2e-05,
+      "loss": 0.935,
+      "step": 2330
+    },
+    {
+      "epoch": 0.2100727174791274,
+      "grad_norm": 0.6760455369949341,
+      "learning_rate": 2e-05,
+      "loss": 0.9143,
+      "step": 2340
+    },
+    {
+      "epoch": 0.2109704641350211,
+      "grad_norm": 0.48834580183029175,
+      "learning_rate": 2e-05,
+      "loss": 0.9758,
+      "step": 2350
+    },
+    {
+      "epoch": 0.2118682107909148,
+      "grad_norm": 0.5034499764442444,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2360
+    },
+    {
+      "epoch": 0.2127659574468085,
+      "grad_norm": 0.47994714975357056,
+      "learning_rate": 2e-05,
+      "loss": 0.9517,
+      "step": 2370
+    },
+    {
+      "epoch": 0.21366370410270222,
+      "grad_norm": 0.42571839690208435,
+      "learning_rate": 2e-05,
+      "loss": 0.9645,
+      "step": 2380
+    },
+    {
+      "epoch": 0.21456145075859592,
+      "grad_norm": 0.49876219034194946,
+      "learning_rate": 2e-05,
+      "loss": 0.9437,
+      "step": 2390
+    },
+    {
+      "epoch": 0.21545919741448963,
+      "grad_norm": 0.46612128615379333,
+      "learning_rate": 2e-05,
+      "loss": 0.9414,
+      "step": 2400
+    },
+    {
+      "epoch": 0.21635694407038333,
+      "grad_norm": 0.4798942804336548,
+      "learning_rate": 2e-05,
+      "loss": 0.8928,
+      "step": 2410
+    },
+    {
+      "epoch": 0.21725469072627704,
+      "grad_norm": 0.4814785420894623,
+      "learning_rate": 2e-05,
+      "loss": 0.957,
+      "step": 2420
+    },
+    {
+      "epoch": 0.21815243738217074,
+      "grad_norm": 0.45460546016693115,
+      "learning_rate": 2e-05,
+      "loss": 0.9066,
+      "step": 2430
+    },
+    {
+      "epoch": 0.21905018403806445,
+      "grad_norm": 0.4495159387588501,
+      "learning_rate": 2e-05,
+      "loss": 0.9417,
+      "step": 2440
+    },
+    {
+      "epoch": 0.21994793069395815,
+      "grad_norm": 0.5322041511535645,
+      "learning_rate": 2e-05,
+      "loss": 0.9186,
+      "step": 2450
+    },
+    {
+      "epoch": 0.22084567734985186,
+      "grad_norm": 0.4809408187866211,
+      "learning_rate": 2e-05,
+      "loss": 0.9293,
+      "step": 2460
+    },
+    {
+      "epoch": 0.22174342400574557,
+      "grad_norm": 0.48313039541244507,
+      "learning_rate": 2e-05,
+      "loss": 0.9358,
+      "step": 2470
+    },
+    {
+      "epoch": 0.2226411706616393,
+      "grad_norm": 0.5264012217521667,
+      "learning_rate": 2e-05,
+      "loss": 0.9699,
+      "step": 2480
+    },
+    {
+      "epoch": 0.223538917317533,
+      "grad_norm": 0.4506959021091461,
+      "learning_rate": 2e-05,
+      "loss": 0.9113,
+      "step": 2490
+    },
+    {
+      "epoch": 0.2244366639734267,
+      "grad_norm": 0.5064295530319214,
+      "learning_rate": 2e-05,
+      "loss": 0.9801,
+      "step": 2500
+    },
+    {
+      "epoch": 0.22533441062932041,
+      "grad_norm": 3.251514196395874,
+      "learning_rate": 2e-05,
+      "loss": 0.9402,
+      "step": 2510
+    },
+    {
+      "epoch": 0.22623215728521412,
+      "grad_norm": 0.6295604109764099,
+      "learning_rate": 2e-05,
+      "loss": 0.9179,
+      "step": 2520
+    },
+    {
+      "epoch": 0.22712990394110782,
+      "grad_norm": 0.45736199617385864,
+      "learning_rate": 2e-05,
+      "loss": 0.8863,
+      "step": 2530
+    },
+    {
+      "epoch": 0.22802765059700153,
+      "grad_norm": 0.5029098987579346,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2540
+    },
+    {
+      "epoch": 0.22892539725289524,
+      "grad_norm": 0.46010321378707886,
+      "learning_rate": 1.9980979278401095e-05,
+      "loss": 0.9845,
+      "step": 2550
+    },
+    {
+      "epoch": 0.22982314390878894,
+      "grad_norm": 0.6344700455665588,
+      "learning_rate": 1.982924955463327e-05,
+      "loss": 0.9732,
+      "step": 2560
+    },
+    {
+      "epoch": 0.23072089056468265,
+      "grad_norm": 0.46013396978378296,
+      "learning_rate": 1.9528108298328235e-05,
+      "loss": 0.9493,
+      "step": 2570
+    },
+    {
+      "epoch": 0.23161863722057635,
+      "grad_norm": 0.573440432548523,
+      "learning_rate": 1.9082156473635994e-05,
+      "loss": 0.9959,
+      "step": 2580
+    },
+    {
+      "epoch": 0.23251638387647006,
+      "grad_norm": 0.5168393850326538,
+      "learning_rate": 1.8498207522139434e-05,
+      "loss": 0.9221,
+      "step": 2590
+    },
+    {
+      "epoch": 0.23341413053236376,
+      "grad_norm": 0.4369233250617981,
+      "learning_rate": 1.7785183264196977e-05,
+      "loss": 0.9243,
+      "step": 2600
+    },
+    {
+      "epoch": 0.23431187718825747,
+      "grad_norm": 0.4613836109638214,
+      "learning_rate": 1.6953977587577846e-05,
+      "loss": 0.9099,
+      "step": 2610
+    },
+    {
+      "epoch": 0.23520962384415117,
+      "grad_norm": 0.5105156302452087,
+      "learning_rate": 1.601729000601297e-05,
+      "loss": 0.9239,
+      "step": 2620
+    },
+    {
+      "epoch": 0.23610737050004488,
+      "grad_norm": 0.4637329578399658,
+      "learning_rate": 1.4989431630624763e-05,
+      "loss": 0.8956,
+      "step": 2630
+    },
+    {
+      "epoch": 0.23700511715593858,
+      "grad_norm": 0.4582845866680145,
+      "learning_rate": 1.3886106518686746e-05,
+      "loss": 0.9649,
+      "step": 2640
+    },
+    {
+      "epoch": 0.2379028638118323,
+      "grad_norm": 0.4612683951854706,
+      "learning_rate": 1.2724171740359612e-05,
+      "loss": 0.8911,
+      "step": 2650
+    },
+    {
+      "epoch": 0.23880061046772602,
+      "grad_norm": 0.5064176321029663,
+      "learning_rate": 1.1521379829206218e-05,
+      "loss": 0.8959,
+      "step": 2660
+    },
+    {
+      "epoch": 0.23969835712361973,
+      "grad_norm": 0.4439590275287628,
+      "learning_rate": 1.029610755143595e-05,
+      "loss": 0.8549,
+      "step": 2670
+    },
+    {
+      "epoch": 0.24059610377951343,
+      "grad_norm": 0.5092665553092957,
+      "learning_rate": 9.067075137857348e-06,
+      "loss": 0.9061,
+      "step": 2680
+    },
+    {
+      "epoch": 0.24149385043540714,
+      "grad_norm": 0.5777043700218201,
+      "learning_rate": 7.853060268232525e-06,
+      "loss": 0.9861,
+      "step": 2690
+    },
+    {
+      "epoch": 0.24239159709130084,
+      "grad_norm": 0.4530249834060669,
+      "learning_rate": 6.672611177902054e-06,
+      "loss": 0.9101,
+      "step": 2700
+    },
+    {
+      "epoch": 0.24328934374719455,
+      "grad_norm": 0.5415931940078735,
+      "learning_rate": 5.543763269959312e-06,
+      "loss": 0.9696,
+      "step": 2710
+    },
+    {
+      "epoch": 0.24418709040308825,
+      "grad_norm": 0.5768683552742004,
+      "learning_rate": 4.483763562693867e-06,
+      "loss": 0.9557,
+      "step": 2720
+    },
+    {
+      "epoch": 0.24508483705898196,
+      "grad_norm": 0.47318390011787415,
+      "learning_rate": 3.508807182313187e-06,
+      "loss": 0.8969,
+      "step": 2730
+    },
+    {
+      "epoch": 0.24598258371487566,
+      "grad_norm": 0.42830783128738403,
+      "learning_rate": 2.6337899269187294e-06,
+      "loss": 0.8998,
+      "step": 2740
+    },
+    {
+      "epoch": 0.24688033037076937,
+      "grad_norm": 0.48739808797836304,
+      "learning_rate": 1.8720806821693126e-06,
+      "loss": 0.9262,
+      "step": 2750
+    },
+    {
+      "epoch": 0.24777807702666307,
+      "grad_norm": 0.4326620101928711,
+      "learning_rate": 1.2353171657619887e-06,
+      "loss": 0.9153,
+      "step": 2760
+    },
+    {
+      "epoch": 0.24867582368255678,
+      "grad_norm": 0.464107483625412,
+      "learning_rate": 7.332281214330356e-07,
+      "loss": 0.8894,
+      "step": 2770
+    },
+    {
+      "epoch": 0.24957357033845048,
+      "grad_norm": 0.4274747371673584,
+      "learning_rate": 3.73484679074808e-07,
+      "loss": 0.896,
+      "step": 2780
+    },
+    {
+      "epoch": 0.2504713169943442,
+      "grad_norm": 0.4573462903499603,
+      "learning_rate": 1.61583151951702e-07,
+      "loss": 0.8731,
+      "step": 2790
+    },
+    {
+      "epoch": 0.25118951431905917,
+      "step": 2798,
+      "total_flos": 1.1366756130249245e+19,
+      "train_loss": 0.08442884704911598,
+      "train_runtime": 2296.0802,
+      "train_samples_per_second": 14.623,
+      "train_steps_per_second": 1.219
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2798,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1366756130249245e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g1_blog_post__edit/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9476001015830018,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.005595283442700529,
+    "train_runtime": 605.769,
+    "train_samples_per_second": 221.748,
+    "train_steps_per_second": 18.479
+}

g1_blog_post__edit/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_blog_post__edit/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_blog_post__edit/1000M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_blog_post__edit/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_blog_post__edit/1000M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_blog_post__edit/1000M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_blog_post__edit/1000M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9476001015830018,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.005595283442700529,
+    "train_runtime": 605.769,
+    "train_samples_per_second": 221.748,
+    "train_steps_per_second": 18.479
+}

g1_blog_post__edit/1000M/trainer_log.jsonl

@@ -0,0 +1,109 @@
+{"current_steps": 10180, "total_steps": 11194, "loss": 0.9821, "lr": 1.999957358086795e-05, "epoch": 0.8617624650808431, "percentage": 90.94, "elapsed_time": "0:00:37", "remaining_time": "0:00:03"}
+{"current_steps": 10190, "total_steps": 11194, "loss": 1.0026, "lr": 1.9991993813020483e-05, "epoch": 0.8626089900956573, "percentage": 91.03, "elapsed_time": "0:02:06", "remaining_time": "0:00:12"}
+{"current_steps": 10200, "total_steps": 11194, "loss": 1.014, "lr": 1.9974946373028628e-05, "epoch": 0.8634555151104715, "percentage": 91.12, "elapsed_time": "0:03:34", "remaining_time": "0:00:20"}
+{"current_steps": 10210, "total_steps": 11194, "loss": 1.0102, "lr": 1.994844749501764e-05, "epoch": 0.8643020401252857, "percentage": 91.21, "elapsed_time": "0:05:03", "remaining_time": "0:00:29"}
+{"current_steps": 10220, "total_steps": 11194, "loss": 1.0104, "lr": 1.9912522413632988e-05, "epoch": 0.8651485651400999, "percentage": 91.3, "elapsed_time": "0:06:32", "remaining_time": "0:00:37"}
+{"current_steps": 10230, "total_steps": 11194, "loss": 0.9994, "lr": 1.9867205340009656e-05, "epoch": 0.865995090154914, "percentage": 91.39, "elapsed_time": "0:08:01", "remaining_time": "0:00:45"}
+{"current_steps": 10240, "total_steps": 11194, "loss": 0.9946, "lr": 1.9812539429193164e-05, "epoch": 0.8668416151697282, "percentage": 91.48, "elapsed_time": "0:09:29", "remaining_time": "0:00:53"}
+{"current_steps": 10250, "total_steps": 11194, "loss": 0.9996, "lr": 1.9748576739043423e-05, "epoch": 0.8676881401845424, "percentage": 91.57, "elapsed_time": "0:10:58", "remaining_time": "0:01:00"}
+{"current_steps": 10260, "total_steps": 11194, "loss": 0.9769, "lr": 1.967537818066047e-05, "epoch": 0.8685346651993566, "percentage": 91.66, "elapsed_time": "0:12:27", "remaining_time": "0:01:08"}
+{"current_steps": 10270, "total_steps": 11194, "loss": 0.9962, "lr": 1.959301346037941e-05, "epoch": 0.8693811902141708, "percentage": 91.75, "elapsed_time": "0:13:56", "remaining_time": "0:01:15"}
+{"current_steps": 10280, "total_steps": 11194, "loss": 0.9871, "lr": 1.9501561013389646e-05, "epoch": 0.870227715228985, "percentage": 91.83, "elapsed_time": "0:15:25", "remaining_time": "0:01:22"}
+{"current_steps": 10290, "total_steps": 11194, "loss": 1.0185, "lr": 1.9401107929041795e-05, "epoch": 0.8710742402437992, "percentage": 91.92, "elapsed_time": "0:16:53", "remaining_time": "0:01:29"}
+{"current_steps": 10300, "total_steps": 11194, "loss": 0.9835, "lr": 1.9291749867913225e-05, "epoch": 0.8719207652586134, "percentage": 92.01, "elapsed_time": "0:18:22", "remaining_time": "0:01:35"}
+{"current_steps": 10310, "total_steps": 11194, "loss": 1.0234, "lr": 1.91735909707114e-05, "epoch": 0.8727672902734276, "percentage": 92.1, "elapsed_time": "0:19:51", "remaining_time": "0:01:42"}
+{"current_steps": 10320, "total_steps": 11194, "loss": 0.9932, "lr": 1.9046743759101545e-05, "epoch": 0.8736138152882418, "percentage": 92.19, "elapsed_time": "0:21:20", "remaining_time": "0:01:48"}
+{"current_steps": 10330, "total_steps": 11194, "loss": 1.0059, "lr": 1.8911329028553324e-05, "epoch": 0.874460340303056, "percentage": 92.28, "elapsed_time": "0:22:49", "remaining_time": "0:01:54"}
+{"current_steps": 10340, "total_steps": 11194, "loss": 1.0, "lr": 1.876747573330835e-05, "epoch": 0.8753068653178702, "percentage": 92.37, "elapsed_time": "0:24:56", "remaining_time": "0:02:03"}
+{"current_steps": 10350, "total_steps": 11194, "loss": 0.998, "lr": 1.8615320863578244e-05, "epoch": 0.8761533903326844, "percentage": 92.46, "elapsed_time": "0:26:25", "remaining_time": "0:02:09"}
+{"current_steps": 10360, "total_steps": 11194, "loss": 0.996, "lr": 1.8455009315090036e-05, "epoch": 0.8769999153474985, "percentage": 92.55, "elapsed_time": "0:27:53", "remaining_time": "0:02:14"}
+{"current_steps": 10370, "total_steps": 11194, "loss": 1.0072, "lr": 1.8286693751103294e-05, "epoch": 0.8778464403623127, "percentage": 92.64, "elapsed_time": "0:29:22", "remaining_time": "0:02:20"}
+{"current_steps": 10380, "total_steps": 11194, "loss": 1.014, "lr": 1.811053445703024e-05, "epoch": 0.8786929653771269, "percentage": 92.73, "elapsed_time": "0:30:51", "remaining_time": "0:02:25"}
+{"current_steps": 10390, "total_steps": 11194, "loss": 1.0361, "lr": 1.7926699187797375e-05, "epoch": 0.8795394903919411, "percentage": 92.82, "elapsed_time": "0:32:19", "remaining_time": "0:02:30"}
+{"current_steps": 10400, "total_steps": 11194, "loss": 1.0111, "lr": 1.7735363008093995e-05, "epoch": 0.8803860154067553, "percentage": 92.91, "elapsed_time": "0:33:48", "remaining_time": "0:02:34"}
+{"current_steps": 10410, "total_steps": 11194, "loss": 1.0178, "lr": 1.7536708125659577e-05, "epoch": 0.8812325404215695, "percentage": 93.0, "elapsed_time": "0:35:17", "remaining_time": "0:02:39"}
+{"current_steps": 10420, "total_steps": 11194, "loss": 1.0143, "lr": 1.733092371776905e-05, "epoch": 0.8820790654363836, "percentage": 93.09, "elapsed_time": "0:36:45", "remaining_time": "0:02:43"}
+{"current_steps": 10430, "total_steps": 11194, "loss": 0.9928, "lr": 1.7118205751080915e-05, "epoch": 0.8829255904511978, "percentage": 93.17, "elapsed_time": "0:38:14", "remaining_time": "0:02:48"}
+{"current_steps": 10440, "total_steps": 11194, "loss": 1.0278, "lr": 1.6898756795019935e-05, "epoch": 0.883772115466012, "percentage": 93.26, "elapsed_time": "0:39:43", "remaining_time": "0:02:52"}
+{"current_steps": 10450, "total_steps": 11194, "loss": 1.018, "lr": 1.667278582887211e-05, "epoch": 0.8846186404808262, "percentage": 93.35, "elapsed_time": "0:41:11", "remaining_time": "0:02:55"}
+{"current_steps": 10460, "total_steps": 11194, "loss": 0.9786, "lr": 1.6440508042775545e-05, "epoch": 0.8854651654956404, "percentage": 93.44, "elapsed_time": "0:42:40", "remaining_time": "0:02:59"}
+{"current_steps": 10470, "total_steps": 11194, "loss": 1.0073, "lr": 1.6202144632796796e-05, "epoch": 0.8863116905104546, "percentage": 93.53, "elapsed_time": "0:44:09", "remaining_time": "0:03:03"}
+{"current_steps": 10480, "total_steps": 11194, "loss": 0.9962, "lr": 1.595792259028783e-05, "epoch": 0.8871582155252687, "percentage": 93.62, "elapsed_time": "0:45:38", "remaining_time": "0:03:06"}
+{"current_steps": 10490, "total_steps": 11194, "loss": 1.0004, "lr": 1.570807448572419e-05, "epoch": 0.8880047405400829, "percentage": 93.71, "elapsed_time": "0:47:07", "remaining_time": "0:03:09"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.9819, "lr": 1.5452838247230197e-05, "epoch": 0.8888512655548971, "percentage": 93.8, "elapsed_time": "0:49:13", "remaining_time": "0:03:15"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.9808, "lr": 1.5192456934002106e-05, "epoch": 0.8896977905697113, "percentage": 93.89, "elapsed_time": "0:50:42", "remaining_time": "0:03:18"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.9932, "lr": 1.492717850484502e-05, "epoch": 0.8905443155845255, "percentage": 93.98, "elapsed_time": "0:52:11", "remaining_time": "0:03:20"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.966, "lr": 1.5452838247230197e-05, "epoch": 0.8888512655548971, "percentage": 93.8, "elapsed_time": "0:00:56", "remaining_time": "0:00:03"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.9808, "lr": 1.5192456934002106e-05, "epoch": 0.8896977905697113, "percentage": 93.89, "elapsed_time": "0:02:24", "remaining_time": "0:00:09"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.9932, "lr": 1.492717850484502e-05, "epoch": 0.8905443155845255, "percentage": 93.98, "elapsed_time": "0:03:53", "remaining_time": "0:00:14"}
+{"current_steps": 10530, "total_steps": 11194, "loss": 1.0116, "lr": 1.4657255582043904e-05, "epoch": 0.8913908405993397, "percentage": 94.07, "elapsed_time": "0:05:21", "remaining_time": "0:00:20"}
+{"current_steps": 10540, "total_steps": 11194, "loss": 1.0224, "lr": 1.4382945210793663e-05, "epoch": 0.8922373656141539, "percentage": 94.16, "elapsed_time": "0:06:50", "remaining_time": "0:00:25"}
+{"current_steps": 10550, "total_steps": 11194, "loss": 0.9648, "lr": 1.4104508614417267e-05, "epoch": 0.8930838906289681, "percentage": 94.25, "elapsed_time": "0:08:18", "remaining_time": "0:00:30"}
+{"current_steps": 10560, "total_steps": 11194, "loss": 1.0074, "lr": 1.382221094560512e-05, "epoch": 0.8939304156437823, "percentage": 94.34, "elapsed_time": "0:09:47", "remaining_time": "0:00:35"}
+{"current_steps": 10570, "total_steps": 11194, "loss": 0.9847, "lr": 1.3536321033912506e-05, "epoch": 0.8947769406585965, "percentage": 94.43, "elapsed_time": "0:11:15", "remaining_time": "0:00:39"}
+{"current_steps": 10580, "total_steps": 11194, "loss": 0.9921, "lr": 1.3247111129755609e-05, "epoch": 0.8956234656734107, "percentage": 94.51, "elapsed_time": "0:12:44", "remaining_time": "0:00:44"}
+{"current_steps": 10590, "total_steps": 11194, "loss": 1.0146, "lr": 1.2954856645149868e-05, "epoch": 0.8964699906882249, "percentage": 94.6, "elapsed_time": "0:14:13", "remaining_time": "0:00:48"}
+{"current_steps": 10600, "total_steps": 11194, "loss": 0.9906, "lr": 1.2659835891437553e-05, "epoch": 0.897316515703039, "percentage": 94.69, "elapsed_time": "0:15:41", "remaining_time": "0:00:52"}
+{"current_steps": 10610, "total_steps": 11194, "loss": 0.9864, "lr": 1.236232981425436e-05, "epoch": 0.8981630407178532, "percentage": 94.78, "elapsed_time": "0:17:10", "remaining_time": "0:00:56"}
+{"current_steps": 10620, "total_steps": 11194, "loss": 0.9841, "lr": 1.2062621725987402e-05, "epoch": 0.8990095657326674, "percentage": 94.87, "elapsed_time": "0:18:38", "remaining_time": "0:01:00"}
+{"current_steps": 10630, "total_steps": 11194, "loss": 0.9754, "lr": 1.1760997035979318e-05, "epoch": 0.8998560907474816, "percentage": 94.96, "elapsed_time": "0:20:07", "remaining_time": "0:01:04"}
+{"current_steps": 10640, "total_steps": 11194, "loss": 0.9914, "lr": 1.1457742978735514e-05, "epoch": 0.9007026157622958, "percentage": 95.05, "elapsed_time": "0:21:35", "remaining_time": "0:01:07"}
+{"current_steps": 10650, "total_steps": 11194, "loss": 0.9988, "lr": 1.1153148340393298e-05, "epoch": 0.90154914077711, "percentage": 95.14, "elapsed_time": "0:23:04", "remaining_time": "0:01:10"}
+{"current_steps": 10660, "total_steps": 11194, "loss": 0.9796, "lr": 1.0847503183713424e-05, "epoch": 0.9023956657919242, "percentage": 95.23, "elapsed_time": "0:25:11", "remaining_time": "0:01:15"}
+{"current_steps": 10670, "total_steps": 11194, "loss": 0.993, "lr": 1.0541098571855896e-05, "epoch": 0.9032421908067383, "percentage": 95.32, "elapsed_time": "0:26:40", "remaining_time": "0:01:18"}
+{"current_steps": 10680, "total_steps": 11194, "loss": 0.9759, "lr": 1.0234226291203123e-05, "epoch": 0.9040887158215525, "percentage": 95.41, "elapsed_time": "0:28:08", "remaining_time": "0:01:21"}
+{"current_steps": 10690, "total_steps": 11194, "loss": 1.0155, "lr": 9.927178573494333e-06, "epoch": 0.9049352408363667, "percentage": 95.5, "elapsed_time": "0:29:37", "remaining_time": "0:01:23"}
+{"current_steps": 10700, "total_steps": 11194, "loss": 0.9985, "lr": 9.620247817535902e-06, "epoch": 0.9057817658511809, "percentage": 95.59, "elapsed_time": "0:31:05", "remaining_time": "0:01:26"}
+{"current_steps": 10710, "total_steps": 11194, "loss": 0.9908, "lr": 9.313726310752581e-06, "epoch": 0.9066282908659951, "percentage": 95.68, "elapsed_time": "0:32:34", "remaining_time": "0:01:28"}
+{"current_steps": 10720, "total_steps": 11194, "loss": 0.997, "lr": 9.00790595084477e-06, "epoch": 0.9074748158808092, "percentage": 95.77, "elapsed_time": "0:34:02", "remaining_time": "0:01:30"}
+{"current_steps": 10730, "total_steps": 11194, "loss": 1.004, "lr": 8.703077967816944e-06, "epoch": 0.9083213408956234, "percentage": 95.85, "elapsed_time": "0:35:31", "remaining_time": "0:01:32"}
+{"current_steps": 10740, "total_steps": 11194, "loss": 0.981, "lr": 8.39953264664192e-06, "epoch": 0.9091678659104376, "percentage": 95.94, "elapsed_time": "0:36:59", "remaining_time": "0:01:33"}
+{"current_steps": 10750, "total_steps": 11194, "loss": 1.0025, "lr": 8.097559050825058e-06, "epoch": 0.9100143909252518, "percentage": 96.03, "elapsed_time": "0:38:28", "remaining_time": "0:01:35"}
+{"current_steps": 10760, "total_steps": 11194, "loss": 1.0045, "lr": 7.79744474713166e-06, "epoch": 0.910860915940066, "percentage": 96.12, "elapsed_time": "0:39:57", "remaining_time": "0:01:36"}
+{"current_steps": 10770, "total_steps": 11194, "loss": 0.9918, "lr": 7.499475531739696e-06, "epoch": 0.9117074409548802, "percentage": 96.21, "elapsed_time": "0:41:25", "remaining_time": "0:01:37"}
+{"current_steps": 10780, "total_steps": 11194, "loss": 0.9843, "lr": 7.203935158078662e-06, "epoch": 0.9125539659696944, "percentage": 96.3, "elapsed_time": "0:42:54", "remaining_time": "0:01:38"}
+{"current_steps": 10790, "total_steps": 11194, "loss": 0.9847, "lr": 6.911105066613709e-06, "epoch": 0.9134004909845086, "percentage": 96.39, "elapsed_time": "0:44:22", "remaining_time": "0:01:39"}
+{"current_steps": 10800, "total_steps": 11194, "loss": 0.995, "lr": 6.621264116832394e-06, "epoch": 0.9142470159993228, "percentage": 96.48, "elapsed_time": "0:45:51", "remaining_time": "0:01:40"}
+{"current_steps": 10810, "total_steps": 11194, "loss": 0.9907, "lr": 6.334688321689274e-06, "epoch": 0.915093541014137, "percentage": 96.57, "elapsed_time": "0:47:19", "remaining_time": "0:01:40"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.986, "lr": 6.051650584761237e-06, "epoch": 0.9159400660289512, "percentage": 96.66, "elapsed_time": "0:49:27", "remaining_time": "0:01:42"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 1.0013, "lr": 5.772420440363841e-06, "epoch": 0.9167865910437654, "percentage": 96.75, "elapsed_time": "0:50:55", "remaining_time": "0:01:42"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.9809, "lr": 6.051650584761237e-06, "epoch": 0.9159400660289512, "percentage": 96.66, "elapsed_time": "0:01:13", "remaining_time": "0:00:02"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 1.0013, "lr": 5.772420440363841e-06, "epoch": 0.9167865910437654, "percentage": 96.75, "elapsed_time": "0:02:42", "remaining_time": "0:00:05"}
+{"current_steps": 10840, "total_steps": 11194, "loss": 0.9963, "lr": 5.497263796876207e-06, "epoch": 0.9176331160585796, "percentage": 96.84, "elapsed_time": "0:04:11", "remaining_time": "0:00:08"}
+{"current_steps": 10850, "total_steps": 11194, "loss": 1.004, "lr": 5.226442683518813e-06, "epoch": 0.9184796410733937, "percentage": 96.93, "elapsed_time": "0:05:40", "remaining_time": "0:00:10"}
+{"current_steps": 10860, "total_steps": 11194, "loss": 0.9681, "lr": 4.960215000825432e-06, "epoch": 0.9193261660882079, "percentage": 97.02, "elapsed_time": "0:07:09", "remaining_time": "0:00:13"}
+{"current_steps": 10870, "total_steps": 11194, "loss": 0.9698, "lr": 4.698834275046711e-06, "epoch": 0.9201726911030221, "percentage": 97.11, "elapsed_time": "0:08:38", "remaining_time": "0:00:15"}
+{"current_steps": 10880, "total_steps": 11194, "loss": 0.9692, "lr": 4.442549416719423e-06, "epoch": 0.9210192161178363, "percentage": 97.19, "elapsed_time": "0:10:07", "remaining_time": "0:00:17"}
+{"current_steps": 10890, "total_steps": 11194, "loss": 1.0016, "lr": 4.1916044836311565e-06, "epoch": 0.9218657411326505, "percentage": 97.28, "elapsed_time": "0:11:36", "remaining_time": "0:00:19"}
+{"current_steps": 10900, "total_steps": 11194, "loss": 1.0105, "lr": 3.94623844840626e-06, "epoch": 0.9227122661474647, "percentage": 97.37, "elapsed_time": "0:13:04", "remaining_time": "0:00:21"}
+{"current_steps": 10910, "total_steps": 11194, "loss": 0.9905, "lr": 3.7066849709343467e-06, "epoch": 0.9235587911622788, "percentage": 97.46, "elapsed_time": "0:14:33", "remaining_time": "0:00:22"}
+{"current_steps": 10920, "total_steps": 11194, "loss": 0.9832, "lr": 3.473172175858059e-06, "epoch": 0.924405316177093, "percentage": 97.55, "elapsed_time": "0:16:02", "remaining_time": "0:00:24"}
+{"current_steps": 10930, "total_steps": 11194, "loss": 0.9824, "lr": 3.2459224353320046e-06, "epoch": 0.9252518411919072, "percentage": 97.64, "elapsed_time": "0:17:31", "remaining_time": "0:00:25"}
+{"current_steps": 10940, "total_steps": 11194, "loss": 0.9736, "lr": 3.025152157259735e-06, "epoch": 0.9260983662067214, "percentage": 97.73, "elapsed_time": "0:19:00", "remaining_time": "0:00:26"}
+{"current_steps": 10950, "total_steps": 11194, "loss": 0.9962, "lr": 2.811071579210425e-06, "epoch": 0.9269448912215356, "percentage": 97.82, "elapsed_time": "0:20:29", "remaining_time": "0:00:27"}
+{"current_steps": 10960, "total_steps": 11194, "loss": 0.9808, "lr": 2.603884568211513e-06, "epoch": 0.9277914162363498, "percentage": 97.91, "elapsed_time": "0:21:58", "remaining_time": "0:00:28"}
+{"current_steps": 10970, "total_steps": 11194, "loss": 0.9711, "lr": 2.4037884266079434e-06, "epoch": 0.928637941251164, "percentage": 98.0, "elapsed_time": "0:23:27", "remaining_time": "0:00:28"}
+{"current_steps": 10980, "total_steps": 11194, "loss": 0.9943, "lr": 2.2109737041728955e-06, "epoch": 0.9294844662659781, "percentage": 98.09, "elapsed_time": "0:25:34", "remaining_time": "0:00:29"}
+{"current_steps": 10990, "total_steps": 11194, "loss": 0.9821, "lr": 2.0256240166489454e-06, "epoch": 0.9303309912807923, "percentage": 98.18, "elapsed_time": "0:27:03", "remaining_time": "0:00:30"}
+{"current_steps": 11000, "total_steps": 11194, "loss": 0.9894, "lr": 1.847915870892413e-06, "epoch": 0.9311775162956065, "percentage": 98.27, "elapsed_time": "0:28:31", "remaining_time": "0:00:30"}
+{"current_steps": 11010, "total_steps": 11194, "loss": 0.9953, "lr": 1.6780184967874715e-06, "epoch": 0.9320240413104207, "percentage": 98.36, "elapsed_time": "0:30:00", "remaining_time": "0:00:30"}
+{"current_steps": 11020, "total_steps": 11194, "loss": 0.9388, "lr": 1.5160936860900294e-06, "epoch": 0.9328705663252349, "percentage": 98.45, "elapsed_time": "0:31:29", "remaining_time": "0:00:29"}
+{"current_steps": 11030, "total_steps": 11194, "loss": 0.9889, "lr": 1.3622956383548762e-06, "epoch": 0.9337170913400491, "percentage": 98.53, "elapsed_time": "0:32:58", "remaining_time": "0:00:29"}
+{"current_steps": 11040, "total_steps": 11194, "loss": 1.0168, "lr": 1.216770814092834e-06, "epoch": 0.9345636163548633, "percentage": 98.62, "elapsed_time": "0:34:27", "remaining_time": "0:00:28"}
+{"current_steps": 11050, "total_steps": 11194, "loss": 0.9763, "lr": 1.0796577952977013e-06, "epoch": 0.9354101413696775, "percentage": 98.71, "elapsed_time": "0:35:56", "remaining_time": "0:00:28"}
+{"current_steps": 11060, "total_steps": 11194, "loss": 0.9886, "lr": 9.510871534758705e-07, "epoch": 0.9362566663844917, "percentage": 98.8, "elapsed_time": "0:37:24", "remaining_time": "0:00:27"}
+{"current_steps": 11070, "total_steps": 11194, "loss": 0.9993, "lr": 8.311813253042334e-07, "epoch": 0.9371031913993059, "percentage": 98.89, "elapsed_time": "0:38:53", "remaining_time": "0:00:26"}
+{"current_steps": 11080, "total_steps": 11194, "loss": 0.9651, "lr": 7.200544960348308e-07, "epoch": 0.9379497164141201, "percentage": 98.98, "elapsed_time": "0:40:22", "remaining_time": "0:00:24"}
+{"current_steps": 11090, "total_steps": 11194, "loss": 0.9752, "lr": 6.178124907572545e-07, "epoch": 0.9387962414289343, "percentage": 99.07, "elapsed_time": "0:41:51", "remaining_time": "0:00:23"}
+{"current_steps": 11100, "total_steps": 11194, "loss": 0.9815, "lr": 5.245526736223528e-07, "epoch": 0.9396427664437484, "percentage": 99.16, "elapsed_time": "0:43:20", "remaining_time": "0:00:22"}
+{"current_steps": 11110, "total_steps": 11194, "loss": 0.9746, "lr": 4.403638551232205e-07, "epoch": 0.9404892914585626, "percentage": 99.25, "elapsed_time": "0:44:49", "remaining_time": "0:00:20"}
+{"current_steps": 11120, "total_steps": 11194, "loss": 0.9973, "lr": 3.6532620752175616e-07, "epoch": 0.9413358164733768, "percentage": 99.34, "elapsed_time": "0:46:17", "remaining_time": "0:00:18"}
+{"current_steps": 11130, "total_steps": 11194, "loss": 0.9816, "lr": 2.9951118850133247e-07, "epoch": 0.942182341488191, "percentage": 99.43, "elapsed_time": "0:47:46", "remaining_time": "0:00:16"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.9594, "lr": 2.429814731182774e-07, "epoch": 0.9430288665030052, "percentage": 99.52, "elapsed_time": "0:49:53", "remaining_time": "0:00:14"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.9594, "lr": 2.429814731182774e-07, "epoch": 0.9430288665030052, "percentage": 99.52, "elapsed_time": "0:01:31", "remaining_time": "0:00:00"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.9841, "lr": 1.957908941169813e-07, "epoch": 0.9438753915178194, "percentage": 99.61, "elapsed_time": "0:02:59", "remaining_time": "0:00:00"}
+{"current_steps": 11160, "total_steps": 11194, "loss": 0.9716, "lr": 1.5798439066545254e-07, "epoch": 0.9447219165326335, "percentage": 99.7, "elapsed_time": "0:04:27", "remaining_time": "0:00:00"}
+{"current_steps": 11170, "total_steps": 11194, "loss": 0.9871, "lr": 1.2959796556014471e-07, "epoch": 0.9455684415474477, "percentage": 99.79, "elapsed_time": "0:05:56", "remaining_time": "0:00:00"}
+{"current_steps": 11180, "total_steps": 11194, "loss": 0.9867, "lr": 1.1065865094081884e-07, "epoch": 0.9464149665622619, "percentage": 99.87, "elapsed_time": "0:07:24", "remaining_time": "0:00:00"}
+{"current_steps": 11190, "total_steps": 11194, "loss": 0.9912, "lr": 1.011844825480734e-07, "epoch": 0.9472614915770761, "percentage": 99.96, "elapsed_time": "0:08:52", "remaining_time": "0:00:00"}
+{"current_steps": 11194, "total_steps": 11194, "epoch": 0.9476001015830018, "percentage": 100.0, "elapsed_time": "0:10:05", "remaining_time": "0:00:00"}

g1_blog_post__edit/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23685769914500973,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.09888922034884624,
+    "train_runtime": 2290.2876,
+    "train_samples_per_second": 14.66,
+    "train_steps_per_second": 1.222
+}

g1_blog_post__edit/250M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_blog_post__edit/250M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_blog_post__edit/250M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_blog_post__edit/250M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_blog_post__edit/250M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_blog_post__edit/250M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_blog_post__edit/250M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23685769914500973,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.09888922034884624,
+    "train_runtime": 2290.2876,
+    "train_samples_per_second": 14.66,
+    "train_steps_per_second": 1.222
+}

g1_blog_post__edit/250M/trainer_log.jsonl

@@ -0,0 +1,26 @@
+{"current_steps": 2550, "total_steps": 2798, "loss": 1.0796, "lr": 1.9980979278401095e-05, "epoch": 0.21586387877761787, "percentage": 91.14, "elapsed_time": "0:00:57", "remaining_time": "0:00:05"}
+{"current_steps": 2560, "total_steps": 2798, "loss": 1.1063, "lr": 1.982924955463327e-05, "epoch": 0.21671040379243206, "percentage": 91.49, "elapsed_time": "0:02:25", "remaining_time": "0:00:13"}
+{"current_steps": 2570, "total_steps": 2798, "loss": 1.0726, "lr": 1.9528108298328235e-05, "epoch": 0.21755692880724625, "percentage": 91.85, "elapsed_time": "0:03:54", "remaining_time": "0:00:20"}
+{"current_steps": 2580, "total_steps": 2798, "loss": 1.1088, "lr": 1.9082156473635994e-05, "epoch": 0.21840345382206044, "percentage": 92.21, "elapsed_time": "0:05:23", "remaining_time": "0:00:27"}
+{"current_steps": 2590, "total_steps": 2798, "loss": 1.1222, "lr": 1.8498207522139434e-05, "epoch": 0.21924997883687464, "percentage": 92.57, "elapsed_time": "0:06:51", "remaining_time": "0:00:33"}
+{"current_steps": 2600, "total_steps": 2798, "loss": 1.1029, "lr": 1.7785183264196977e-05, "epoch": 0.22009650385168883, "percentage": 92.92, "elapsed_time": "0:08:20", "remaining_time": "0:00:38"}
+{"current_steps": 2610, "total_steps": 2798, "loss": 1.1041, "lr": 1.6953977587577846e-05, "epoch": 0.220943028866503, "percentage": 93.28, "elapsed_time": "0:09:49", "remaining_time": "0:00:42"}
+{"current_steps": 2620, "total_steps": 2798, "loss": 1.0977, "lr": 1.601729000601297e-05, "epoch": 0.2217895538813172, "percentage": 93.64, "elapsed_time": "0:11:18", "remaining_time": "0:00:46"}
+{"current_steps": 2630, "total_steps": 2798, "loss": 1.109, "lr": 1.4989431630624763e-05, "epoch": 0.22263607889613138, "percentage": 94.0, "elapsed_time": "0:12:46", "remaining_time": "0:00:48"}
+{"current_steps": 2640, "total_steps": 2798, "loss": 1.1031, "lr": 1.3886106518686746e-05, "epoch": 0.22348260391094557, "percentage": 94.35, "elapsed_time": "0:14:15", "remaining_time": "0:00:51"}
+{"current_steps": 2650, "total_steps": 2798, "loss": 1.0749, "lr": 1.2724171740359612e-05, "epoch": 0.22432912892575976, "percentage": 94.71, "elapsed_time": "0:15:44", "remaining_time": "0:00:52"}
+{"current_steps": 2660, "total_steps": 2798, "loss": 1.0914, "lr": 1.1521379829206218e-05, "epoch": 0.22517565394057396, "percentage": 95.07, "elapsed_time": "0:17:13", "remaining_time": "0:00:53"}
+{"current_steps": 2670, "total_steps": 2798, "loss": 1.1083, "lr": 1.029610755143595e-05, "epoch": 0.22602217895538812, "percentage": 95.43, "elapsed_time": "0:18:41", "remaining_time": "0:00:53"}
+{"current_steps": 2680, "total_steps": 2798, "loss": 1.0708, "lr": 9.067075137857348e-06, "epoch": 0.2268687039702023, "percentage": 95.78, "elapsed_time": "0:20:10", "remaining_time": "0:00:53"}
+{"current_steps": 2690, "total_steps": 2798, "loss": 1.0812, "lr": 7.853060268232525e-06, "epoch": 0.2277152289850165, "percentage": 96.14, "elapsed_time": "0:21:39", "remaining_time": "0:00:52"}
+{"current_steps": 2700, "total_steps": 2798, "loss": 1.0822, "lr": 6.672611177902054e-06, "epoch": 0.2285617539998307, "percentage": 96.5, "elapsed_time": "0:23:07", "remaining_time": "0:00:50"}
+{"current_steps": 2710, "total_steps": 2798, "loss": 1.0845, "lr": 5.543763269959312e-06, "epoch": 0.2294082790146449, "percentage": 96.85, "elapsed_time": "0:24:53", "remaining_time": "0:00:48"}
+{"current_steps": 2720, "total_steps": 2798, "loss": 1.0787, "lr": 4.483763562693867e-06, "epoch": 0.23025480402945908, "percentage": 97.21, "elapsed_time": "0:26:21", "remaining_time": "0:00:45"}
+{"current_steps": 2730, "total_steps": 2798, "loss": 1.086, "lr": 3.508807182313187e-06, "epoch": 0.23110132904427325, "percentage": 97.57, "elapsed_time": "0:27:50", "remaining_time": "0:00:41"}
+{"current_steps": 2740, "total_steps": 2798, "loss": 1.0848, "lr": 2.6337899269187294e-06, "epoch": 0.23194785405908744, "percentage": 97.93, "elapsed_time": "0:29:19", "remaining_time": "0:00:37"}
+{"current_steps": 2750, "total_steps": 2798, "loss": 1.0855, "lr": 1.8720806821693126e-06, "epoch": 0.23279437907390163, "percentage": 98.28, "elapsed_time": "0:30:48", "remaining_time": "0:00:32"}
+{"current_steps": 2760, "total_steps": 2798, "loss": 1.0752, "lr": 1.2353171657619887e-06, "epoch": 0.23364090408871582, "percentage": 98.64, "elapsed_time": "0:32:16", "remaining_time": "0:00:26"}
+{"current_steps": 2770, "total_steps": 2798, "loss": 1.0848, "lr": 7.332281214330356e-07, "epoch": 0.23448742910353002, "percentage": 99.0, "elapsed_time": "0:33:45", "remaining_time": "0:00:20"}
+{"current_steps": 2780, "total_steps": 2798, "loss": 1.0935, "lr": 3.73484679074808e-07, "epoch": 0.2353339541183442, "percentage": 99.36, "elapsed_time": "0:35:14", "remaining_time": "0:00:13"}
+{"current_steps": 2790, "total_steps": 2798, "loss": 1.0823, "lr": 1.61583151951702e-07, "epoch": 0.23618047913315837, "percentage": 99.71, "elapsed_time": "0:36:42", "remaining_time": "0:00:06"}
+{"current_steps": 2798, "total_steps": 2798, "epoch": 0.23685769914500973, "percentage": 100.0, "elapsed_time": "0:38:10", "remaining_time": "0:00:00"}

g1_blog_post__edit/250M/trainer_state.json

@@ -0,0 +1,1996 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.23685769914500973,
+  "eval_steps": 500,
+  "global_step": 2798,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008465250148141877,
+      "grad_norm": 1.375163197517395,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.705,
+      "step": 10
+    },
+    {
+      "epoch": 0.0016930500296283755,
+      "grad_norm": 1.2523367404937744,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 1.6253,
+      "step": 20
+    },
+    {
+      "epoch": 0.0025395750444425634,
+      "grad_norm": 1.0373904705047607,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.573,
+      "step": 30
+    },
+    {
+      "epoch": 0.003386100059256751,
+      "grad_norm": 1.1109267473220825,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.4768,
+      "step": 40
+    },
+    {
+      "epoch": 0.004232625074070939,
+      "grad_norm": 1.4183589220046997,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 1.5007,
+      "step": 50
+    },
+    {
+      "epoch": 0.005079150088885127,
+      "grad_norm": 0.6070196032524109,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 1.3877,
+      "step": 60
+    },
+    {
+      "epoch": 0.005925675103699314,
+      "grad_norm": 0.9637130498886108,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 1.3851,
+      "step": 70
+    },
+    {
+      "epoch": 0.006772200118513502,
+      "grad_norm": 0.8439693450927734,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 1.3494,
+      "step": 80
+    },
+    {
+      "epoch": 0.00761872513332769,
+      "grad_norm": 0.6147797107696533,
+      "learning_rate": 1.005e-05,
+      "loss": 1.2999,
+      "step": 90
+    },
+    {
+      "epoch": 0.008465250148141878,
+      "grad_norm": 0.6338775753974915,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 1.2799,
+      "step": 100
+    },
+    {
+      "epoch": 0.009311775162956065,
+      "grad_norm": 0.6608278751373291,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 1.3033,
+      "step": 110
+    },
+    {
+      "epoch": 0.010158300177770254,
+      "grad_norm": 0.6294130682945251,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 1.2796,
+      "step": 120
+    },
+    {
+      "epoch": 0.01100482519258444,
+      "grad_norm": 0.7209584712982178,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 1.2888,
+      "step": 130
+    },
+    {
+      "epoch": 0.011851350207398628,
+      "grad_norm": 0.6430320739746094,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 1.2495,
+      "step": 140
+    },
+    {
+      "epoch": 0.012697875222212817,
+      "grad_norm": 0.6338914632797241,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 1.2607,
+      "step": 150
+    },
+    {
+      "epoch": 0.013544400237027004,
+      "grad_norm": 0.6515811085700989,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 1.2938,
+      "step": 160
+    },
+    {
+      "epoch": 0.014390925251841193,
+      "grad_norm": 0.6354628205299377,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 1.2467,
+      "step": 170
+    },
+    {
+      "epoch": 0.01523745026665538,
+      "grad_norm": 0.640027642250061,
+      "learning_rate": 2e-05,
+      "loss": 1.2315,
+      "step": 180
+    },
+    {
+      "epoch": 0.016083975281469567,
+      "grad_norm": 0.6222153306007385,
+      "learning_rate": 2e-05,
+      "loss": 1.2303,
+      "step": 190
+    },
+    {
+      "epoch": 0.016930500296283756,
+      "grad_norm": 0.6284162402153015,
+      "learning_rate": 2e-05,
+      "loss": 1.2627,
+      "step": 200
+    },
+    {
+      "epoch": 0.017777025311097944,
+      "grad_norm": 0.6227605938911438,
+      "learning_rate": 2e-05,
+      "loss": 1.2303,
+      "step": 210
+    },
+    {
+      "epoch": 0.01862355032591213,
+      "grad_norm": 0.5966870784759521,
+      "learning_rate": 2e-05,
+      "loss": 1.2273,
+      "step": 220
+    },
+    {
+      "epoch": 0.01947007534072632,
+      "grad_norm": 0.6203404664993286,
+      "learning_rate": 2e-05,
+      "loss": 1.2527,
+      "step": 230
+    },
+    {
+      "epoch": 0.020316600355540507,
+      "grad_norm": 0.6143743395805359,
+      "learning_rate": 2e-05,
+      "loss": 1.2345,
+      "step": 240
+    },
+    {
+      "epoch": 0.021163125370354693,
+      "grad_norm": 0.5946654677391052,
+      "learning_rate": 2e-05,
+      "loss": 1.2264,
+      "step": 250
+    },
+    {
+      "epoch": 0.02200965038516888,
+      "grad_norm": 0.6699089407920837,
+      "learning_rate": 2e-05,
+      "loss": 1.2679,
+      "step": 260
+    },
+    {
+      "epoch": 0.02285617539998307,
+      "grad_norm": 0.6096220016479492,
+      "learning_rate": 2e-05,
+      "loss": 1.2619,
+      "step": 270
+    },
+    {
+      "epoch": 0.023702700414797256,
+      "grad_norm": 0.5971185564994812,
+      "learning_rate": 2e-05,
+      "loss": 1.1971,
+      "step": 280
+    },
+    {
+      "epoch": 0.024549225429611445,
+      "grad_norm": 0.6447914838790894,
+      "learning_rate": 2e-05,
+      "loss": 1.2161,
+      "step": 290
+    },
+    {
+      "epoch": 0.025395750444425633,
+      "grad_norm": 0.5698952674865723,
+      "learning_rate": 2e-05,
+      "loss": 1.2239,
+      "step": 300
+    },
+    {
+      "epoch": 0.026242275459239822,
+      "grad_norm": 0.5705931186676025,
+      "learning_rate": 2e-05,
+      "loss": 1.2223,
+      "step": 310
+    },
+    {
+      "epoch": 0.027088800474054008,
+      "grad_norm": 0.5685964822769165,
+      "learning_rate": 2e-05,
+      "loss": 1.2303,
+      "step": 320
+    },
+    {
+      "epoch": 0.027935325488868196,
+      "grad_norm": 0.57871013879776,
+      "learning_rate": 2e-05,
+      "loss": 1.2263,
+      "step": 330
+    },
+    {
+      "epoch": 0.028781850503682385,
+      "grad_norm": 0.6025750637054443,
+      "learning_rate": 2e-05,
+      "loss": 1.2248,
+      "step": 340
+    },
+    {
+      "epoch": 0.02962837551849657,
+      "grad_norm": 0.6065348982810974,
+      "learning_rate": 2e-05,
+      "loss": 1.2059,
+      "step": 350
+    },
+    {
+      "epoch": 0.03047490053331076,
+      "grad_norm": 0.7909114956855774,
+      "learning_rate": 2e-05,
+      "loss": 1.1957,
+      "step": 360
+    },
+    {
+      "epoch": 0.03132142554812495,
+      "grad_norm": 0.6087936162948608,
+      "learning_rate": 2e-05,
+      "loss": 1.2028,
+      "step": 370
+    },
+    {
+      "epoch": 0.032167950562939134,
+      "grad_norm": 0.5729495286941528,
+      "learning_rate": 2e-05,
+      "loss": 1.1988,
+      "step": 380
+    },
+    {
+      "epoch": 0.033014475577753326,
+      "grad_norm": 0.5869688987731934,
+      "learning_rate": 2e-05,
+      "loss": 1.1826,
+      "step": 390
+    },
+    {
+      "epoch": 0.03386100059256751,
+      "grad_norm": 0.5820246934890747,
+      "learning_rate": 2e-05,
+      "loss": 1.2353,
+      "step": 400
+    },
+    {
+      "epoch": 0.034707525607381697,
+      "grad_norm": 0.602737545967102,
+      "learning_rate": 2e-05,
+      "loss": 1.2115,
+      "step": 410
+    },
+    {
+      "epoch": 0.03555405062219589,
+      "grad_norm": 0.6072444915771484,
+      "learning_rate": 2e-05,
+      "loss": 1.1846,
+      "step": 420
+    },
+    {
+      "epoch": 0.036400575637010074,
+      "grad_norm": 0.5940274596214294,
+      "learning_rate": 2e-05,
+      "loss": 1.2124,
+      "step": 430
+    },
+    {
+      "epoch": 0.03724710065182426,
+      "grad_norm": 0.5697258710861206,
+      "learning_rate": 2e-05,
+      "loss": 1.2146,
+      "step": 440
+    },
+    {
+      "epoch": 0.03809362566663845,
+      "grad_norm": 0.573409914970398,
+      "learning_rate": 2e-05,
+      "loss": 1.2034,
+      "step": 450
+    },
+    {
+      "epoch": 0.03894015068145264,
+      "grad_norm": 0.5774071216583252,
+      "learning_rate": 2e-05,
+      "loss": 1.207,
+      "step": 460
+    },
+    {
+      "epoch": 0.03978667569626682,
+      "grad_norm": 0.5621113777160645,
+      "learning_rate": 2e-05,
+      "loss": 1.2152,
+      "step": 470
+    },
+    {
+      "epoch": 0.040633200711081015,
+      "grad_norm": 0.578490138053894,
+      "learning_rate": 2e-05,
+      "loss": 1.1924,
+      "step": 480
+    },
+    {
+      "epoch": 0.0414797257258952,
+      "grad_norm": 0.5755975842475891,
+      "learning_rate": 2e-05,
+      "loss": 1.1881,
+      "step": 490
+    },
+    {
+      "epoch": 0.042326250740709385,
+      "grad_norm": 0.597003161907196,
+      "learning_rate": 2e-05,
+      "loss": 1.218,
+      "step": 500
+    },
+    {
+      "epoch": 0.04317277575552358,
+      "grad_norm": 0.5683897137641907,
+      "learning_rate": 2e-05,
+      "loss": 1.216,
+      "step": 510
+    },
+    {
+      "epoch": 0.04401930077033776,
+      "grad_norm": 0.5480225086212158,
+      "learning_rate": 2e-05,
+      "loss": 1.1749,
+      "step": 520
+    },
+    {
+      "epoch": 0.04486582578515195,
+      "grad_norm": 0.5749838948249817,
+      "learning_rate": 2e-05,
+      "loss": 1.1782,
+      "step": 530
+    },
+    {
+      "epoch": 0.04571235079996614,
+      "grad_norm": 0.5992707014083862,
+      "learning_rate": 2e-05,
+      "loss": 1.1899,
+      "step": 540
+    },
+    {
+      "epoch": 0.046558875814780326,
+      "grad_norm": 0.554719090461731,
+      "learning_rate": 2e-05,
+      "loss": 1.1676,
+      "step": 550
+    },
+    {
+      "epoch": 0.04740540082959451,
+      "grad_norm": 0.5618951320648193,
+      "learning_rate": 2e-05,
+      "loss": 1.2133,
+      "step": 560
+    },
+    {
+      "epoch": 0.048251925844408704,
+      "grad_norm": 0.5630229711532593,
+      "learning_rate": 2e-05,
+      "loss": 1.192,
+      "step": 570
+    },
+    {
+      "epoch": 0.04909845085922289,
+      "grad_norm": 0.5739653706550598,
+      "learning_rate": 2e-05,
+      "loss": 1.1916,
+      "step": 580
+    },
+    {
+      "epoch": 0.049944975874037074,
+      "grad_norm": 0.5743044018745422,
+      "learning_rate": 2e-05,
+      "loss": 1.1839,
+      "step": 590
+    },
+    {
+      "epoch": 0.05079150088885127,
+      "grad_norm": 0.5558018684387207,
+      "learning_rate": 2e-05,
+      "loss": 1.1901,
+      "step": 600
+    },
+    {
+      "epoch": 0.05163802590366545,
+      "grad_norm": 0.5445019006729126,
+      "learning_rate": 2e-05,
+      "loss": 1.1683,
+      "step": 610
+    },
+    {
+      "epoch": 0.052484550918479644,
+      "grad_norm": 0.5387155413627625,
+      "learning_rate": 2e-05,
+      "loss": 1.2137,
+      "step": 620
+    },
+    {
+      "epoch": 0.05333107593329383,
+      "grad_norm": 0.5415998697280884,
+      "learning_rate": 2e-05,
+      "loss": 1.2066,
+      "step": 630
+    },
+    {
+      "epoch": 0.054177600948108015,
+      "grad_norm": 0.5340350866317749,
+      "learning_rate": 2e-05,
+      "loss": 1.1823,
+      "step": 640
+    },
+    {
+      "epoch": 0.05502412596292221,
+      "grad_norm": 0.5698688626289368,
+      "learning_rate": 2e-05,
+      "loss": 1.1855,
+      "step": 650
+    },
+    {
+      "epoch": 0.05587065097773639,
+      "grad_norm": 0.5903260111808777,
+      "learning_rate": 2e-05,
+      "loss": 1.1706,
+      "step": 660
+    },
+    {
+      "epoch": 0.05671717599255058,
+      "grad_norm": 0.5460957288742065,
+      "learning_rate": 2e-05,
+      "loss": 1.1674,
+      "step": 670
+    },
+    {
+      "epoch": 0.05756370100736477,
+      "grad_norm": 0.5396808981895447,
+      "learning_rate": 2e-05,
+      "loss": 1.1708,
+      "step": 680
+    },
+    {
+      "epoch": 0.058410226022178956,
+      "grad_norm": 0.5419284701347351,
+      "learning_rate": 2e-05,
+      "loss": 1.1385,
+      "step": 690
+    },
+    {
+      "epoch": 0.05925675103699314,
+      "grad_norm": 0.5482523441314697,
+      "learning_rate": 2e-05,
+      "loss": 1.1942,
+      "step": 700
+    },
+    {
+      "epoch": 0.06010327605180733,
+      "grad_norm": 0.5543491840362549,
+      "learning_rate": 2e-05,
+      "loss": 1.1852,
+      "step": 710
+    },
+    {
+      "epoch": 0.06094980106662152,
+      "grad_norm": 0.5552930235862732,
+      "learning_rate": 2e-05,
+      "loss": 1.1696,
+      "step": 720
+    },
+    {
+      "epoch": 0.061796326081435704,
+      "grad_norm": 0.5307426452636719,
+      "learning_rate": 2e-05,
+      "loss": 1.1487,
+      "step": 730
+    },
+    {
+      "epoch": 0.0626428510962499,
+      "grad_norm": 0.5562779903411865,
+      "learning_rate": 2e-05,
+      "loss": 1.1896,
+      "step": 740
+    },
+    {
+      "epoch": 0.06348937611106409,
+      "grad_norm": 0.5503221154212952,
+      "learning_rate": 2e-05,
+      "loss": 1.1754,
+      "step": 750
+    },
+    {
+      "epoch": 0.06433590112587827,
+      "grad_norm": 0.5481985211372375,
+      "learning_rate": 2e-05,
+      "loss": 1.1748,
+      "step": 760
+    },
+    {
+      "epoch": 0.06518242614069246,
+      "grad_norm": 0.5233391523361206,
+      "learning_rate": 2e-05,
+      "loss": 1.1767,
+      "step": 770
+    },
+    {
+      "epoch": 0.06602895115550665,
+      "grad_norm": 0.5557997226715088,
+      "learning_rate": 2e-05,
+      "loss": 1.2007,
+      "step": 780
+    },
+    {
+      "epoch": 0.06687547617032083,
+      "grad_norm": 0.5833834409713745,
+      "learning_rate": 2e-05,
+      "loss": 1.1605,
+      "step": 790
+    },
+    {
+      "epoch": 0.06772200118513502,
+      "grad_norm": 0.5553584694862366,
+      "learning_rate": 2e-05,
+      "loss": 1.1611,
+      "step": 800
+    },
+    {
+      "epoch": 0.06856852619994921,
+      "grad_norm": 0.54029780626297,
+      "learning_rate": 2e-05,
+      "loss": 1.1523,
+      "step": 810
+    },
+    {
+      "epoch": 0.06941505121476339,
+      "grad_norm": 0.5397090315818787,
+      "learning_rate": 2e-05,
+      "loss": 1.1707,
+      "step": 820
+    },
+    {
+      "epoch": 0.07026157622957759,
+      "grad_norm": 0.5384632349014282,
+      "learning_rate": 2e-05,
+      "loss": 1.1587,
+      "step": 830
+    },
+    {
+      "epoch": 0.07110810124439178,
+      "grad_norm": 0.5376214385032654,
+      "learning_rate": 2e-05,
+      "loss": 1.1448,
+      "step": 840
+    },
+    {
+      "epoch": 0.07195462625920596,
+      "grad_norm": 0.5435184836387634,
+      "learning_rate": 2e-05,
+      "loss": 1.1654,
+      "step": 850
+    },
+    {
+      "epoch": 0.07280115127402015,
+      "grad_norm": 0.5461548566818237,
+      "learning_rate": 2e-05,
+      "loss": 1.1196,
+      "step": 860
+    },
+    {
+      "epoch": 0.07364767628883434,
+      "grad_norm": 0.5359179377555847,
+      "learning_rate": 2e-05,
+      "loss": 1.1649,
+      "step": 870
+    },
+    {
+      "epoch": 0.07449420130364852,
+      "grad_norm": 0.5784762501716614,
+      "learning_rate": 2e-05,
+      "loss": 1.1274,
+      "step": 880
+    },
+    {
+      "epoch": 0.07534072631846271,
+      "grad_norm": 0.5339529514312744,
+      "learning_rate": 2e-05,
+      "loss": 1.166,
+      "step": 890
+    },
+    {
+      "epoch": 0.0761872513332769,
+      "grad_norm": 0.53230881690979,
+      "learning_rate": 2e-05,
+      "loss": 1.183,
+      "step": 900
+    },
+    {
+      "epoch": 0.07703377634809108,
+      "grad_norm": 0.5454527735710144,
+      "learning_rate": 2e-05,
+      "loss": 1.1577,
+      "step": 910
+    },
+    {
+      "epoch": 0.07788030136290527,
+      "grad_norm": 0.5409994125366211,
+      "learning_rate": 2e-05,
+      "loss": 1.1699,
+      "step": 920
+    },
+    {
+      "epoch": 0.07872682637771947,
+      "grad_norm": 0.5251087546348572,
+      "learning_rate": 2e-05,
+      "loss": 1.177,
+      "step": 930
+    },
+    {
+      "epoch": 0.07957335139253364,
+      "grad_norm": 0.5456949472427368,
+      "learning_rate": 2e-05,
+      "loss": 1.168,
+      "step": 940
+    },
+    {
+      "epoch": 0.08041987640734784,
+      "grad_norm": 0.5512030720710754,
+      "learning_rate": 2e-05,
+      "loss": 1.1576,
+      "step": 950
+    },
+    {
+      "epoch": 0.08126640142216203,
+      "grad_norm": 0.5522686243057251,
+      "learning_rate": 2e-05,
+      "loss": 1.1896,
+      "step": 960
+    },
+    {
+      "epoch": 0.08211292643697621,
+      "grad_norm": 0.5380606055259705,
+      "learning_rate": 2e-05,
+      "loss": 1.1514,
+      "step": 970
+    },
+    {
+      "epoch": 0.0829594514517904,
+      "grad_norm": 0.5439881682395935,
+      "learning_rate": 2e-05,
+      "loss": 1.181,
+      "step": 980
+    },
+    {
+      "epoch": 0.08380597646660459,
+      "grad_norm": 0.539462685585022,
+      "learning_rate": 2e-05,
+      "loss": 1.1596,
+      "step": 990
+    },
+    {
+      "epoch": 0.08465250148141877,
+      "grad_norm": 0.5341487526893616,
+      "learning_rate": 2e-05,
+      "loss": 1.1355,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08549902649623296,
+      "grad_norm": 0.5274637341499329,
+      "learning_rate": 2e-05,
+      "loss": 1.1458,
+      "step": 1010
+    },
+    {
+      "epoch": 0.08634555151104716,
+      "grad_norm": 0.5372074842453003,
+      "learning_rate": 2e-05,
+      "loss": 1.1562,
+      "step": 1020
+    },
+    {
+      "epoch": 0.08719207652586133,
+      "grad_norm": 0.5366864204406738,
+      "learning_rate": 2e-05,
+      "loss": 1.1426,
+      "step": 1030
+    },
+    {
+      "epoch": 0.08803860154067553,
+      "grad_norm": 0.5459463000297546,
+      "learning_rate": 2e-05,
+      "loss": 1.1313,
+      "step": 1040
+    },
+    {
+      "epoch": 0.08888512655548972,
+      "grad_norm": 0.5211199522018433,
+      "learning_rate": 2e-05,
+      "loss": 1.1406,
+      "step": 1050
+    },
+    {
+      "epoch": 0.0897316515703039,
+      "grad_norm": 0.5361355543136597,
+      "learning_rate": 2e-05,
+      "loss": 1.1307,
+      "step": 1060
+    },
+    {
+      "epoch": 0.09057817658511809,
+      "grad_norm": 0.5369033217430115,
+      "learning_rate": 2e-05,
+      "loss": 1.1315,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09142470159993228,
+      "grad_norm": 0.5286295413970947,
+      "learning_rate": 2e-05,
+      "loss": 1.1654,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09227122661474646,
+      "grad_norm": 0.5266904234886169,
+      "learning_rate": 2e-05,
+      "loss": 1.148,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09311775162956065,
+      "grad_norm": 0.5492565035820007,
+      "learning_rate": 2e-05,
+      "loss": 1.1573,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09396427664437484,
+      "grad_norm": 0.5259299874305725,
+      "learning_rate": 2e-05,
+      "loss": 1.1547,
+      "step": 1110
+    },
+    {
+      "epoch": 0.09481080165918902,
+      "grad_norm": 0.5307326316833496,
+      "learning_rate": 2e-05,
+      "loss": 1.138,
+      "step": 1120
+    },
+    {
+      "epoch": 0.09565732667400322,
+      "grad_norm": 0.5256623029708862,
+      "learning_rate": 2e-05,
+      "loss": 1.1363,
+      "step": 1130
+    },
+    {
+      "epoch": 0.09650385168881741,
+      "grad_norm": 0.6133991479873657,
+      "learning_rate": 2e-05,
+      "loss": 1.1668,
+      "step": 1140
+    },
+    {
+      "epoch": 0.09735037670363159,
+      "grad_norm": 0.5217273235321045,
+      "learning_rate": 2e-05,
+      "loss": 1.1648,
+      "step": 1150
+    },
+    {
+      "epoch": 0.09819690171844578,
+      "grad_norm": 0.562288224697113,
+      "learning_rate": 2e-05,
+      "loss": 1.1604,
+      "step": 1160
+    },
+    {
+      "epoch": 0.09904342673325997,
+      "grad_norm": 0.5597480535507202,
+      "learning_rate": 2e-05,
+      "loss": 1.1505,
+      "step": 1170
+    },
+    {
+      "epoch": 0.09988995174807415,
+      "grad_norm": 0.5335237979888916,
+      "learning_rate": 2e-05,
+      "loss": 1.1652,
+      "step": 1180
+    },
+    {
+      "epoch": 0.10073647676288834,
+      "grad_norm": 0.5393009781837463,
+      "learning_rate": 2e-05,
+      "loss": 1.1644,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10158300177770253,
+      "grad_norm": 0.5456632971763611,
+      "learning_rate": 2e-05,
+      "loss": 1.1554,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10242952679251673,
+      "grad_norm": 0.5280405879020691,
+      "learning_rate": 2e-05,
+      "loss": 1.1559,
+      "step": 1210
+    },
+    {
+      "epoch": 0.1032760518073309,
+      "grad_norm": 0.5182194113731384,
+      "learning_rate": 2e-05,
+      "loss": 1.1327,
+      "step": 1220
+    },
+    {
+      "epoch": 0.1041225768221451,
+      "grad_norm": 0.5189276933670044,
+      "learning_rate": 2e-05,
+      "loss": 1.1358,
+      "step": 1230
+    },
+    {
+      "epoch": 0.10496910183695929,
+      "grad_norm": 0.5249096751213074,
+      "learning_rate": 2e-05,
+      "loss": 1.1788,
+      "step": 1240
+    },
+    {
+      "epoch": 0.10581562685177347,
+      "grad_norm": 0.5242507457733154,
+      "learning_rate": 2e-05,
+      "loss": 1.1281,
+      "step": 1250
+    },
+    {
+      "epoch": 0.10666215186658766,
+      "grad_norm": 0.5344493389129639,
+      "learning_rate": 2e-05,
+      "loss": 1.1297,
+      "step": 1260
+    },
+    {
+      "epoch": 0.10750867688140185,
+      "grad_norm": 0.5310614109039307,
+      "learning_rate": 2e-05,
+      "loss": 1.1608,
+      "step": 1270
+    },
+    {
+      "epoch": 0.10835520189621603,
+      "grad_norm": 0.505918025970459,
+      "learning_rate": 2e-05,
+      "loss": 1.1369,
+      "step": 1280
+    },
+    {
+      "epoch": 0.10920172691103022,
+      "grad_norm": 0.5338344573974609,
+      "learning_rate": 2e-05,
+      "loss": 1.1286,
+      "step": 1290
+    },
+    {
+      "epoch": 0.11004825192584441,
+      "grad_norm": 0.499277800321579,
+      "learning_rate": 2e-05,
+      "loss": 1.1494,
+      "step": 1300
+    },
+    {
+      "epoch": 0.1108947769406586,
+      "grad_norm": 0.5376572012901306,
+      "learning_rate": 2e-05,
+      "loss": 1.1475,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11174130195547279,
+      "grad_norm": 0.5293624401092529,
+      "learning_rate": 2e-05,
+      "loss": 1.1219,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11258782697028698,
+      "grad_norm": 0.5140743255615234,
+      "learning_rate": 2e-05,
+      "loss": 1.143,
+      "step": 1330
+    },
+    {
+      "epoch": 0.11343435198510116,
+      "grad_norm": 0.5123709440231323,
+      "learning_rate": 2e-05,
+      "loss": 1.1449,
+      "step": 1340
+    },
+    {
+      "epoch": 0.11428087699991535,
+      "grad_norm": 0.5354812741279602,
+      "learning_rate": 2e-05,
+      "loss": 1.1234,
+      "step": 1350
+    },
+    {
+      "epoch": 0.11512740201472954,
+      "grad_norm": 0.5364473462104797,
+      "learning_rate": 2e-05,
+      "loss": 1.1327,
+      "step": 1360
+    },
+    {
+      "epoch": 0.11597392702954372,
+      "grad_norm": 0.5255948305130005,
+      "learning_rate": 2e-05,
+      "loss": 1.1709,
+      "step": 1370
+    },
+    {
+      "epoch": 0.11682045204435791,
+      "grad_norm": 0.5397413969039917,
+      "learning_rate": 2e-05,
+      "loss": 1.1403,
+      "step": 1380
+    },
+    {
+      "epoch": 0.1176669770591721,
+      "grad_norm": 0.5222242474555969,
+      "learning_rate": 2e-05,
+      "loss": 1.1454,
+      "step": 1390
+    },
+    {
+      "epoch": 0.11851350207398628,
+      "grad_norm": 0.5370845198631287,
+      "learning_rate": 2e-05,
+      "loss": 1.1248,
+      "step": 1400
+    },
+    {
+      "epoch": 0.11936002708880047,
+      "grad_norm": 0.5238989591598511,
+      "learning_rate": 2e-05,
+      "loss": 1.1479,
+      "step": 1410
+    },
+    {
+      "epoch": 0.12020655210361467,
+      "grad_norm": 0.5254695415496826,
+      "learning_rate": 2e-05,
+      "loss": 1.1493,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12105307711842885,
+      "grad_norm": 0.506896436214447,
+      "learning_rate": 2e-05,
+      "loss": 1.0917,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12189960213324304,
+      "grad_norm": 0.5210740566253662,
+      "learning_rate": 2e-05,
+      "loss": 1.1203,
+      "step": 1440
+    },
+    {
+      "epoch": 0.12274612714805723,
+      "grad_norm": 0.5506751537322998,
+      "learning_rate": 2e-05,
+      "loss": 1.1147,
+      "step": 1450
+    },
+    {
+      "epoch": 0.12359265216287141,
+      "grad_norm": 0.5313059091567993,
+      "learning_rate": 2e-05,
+      "loss": 1.151,
+      "step": 1460
+    },
+    {
+      "epoch": 0.1244391771776856,
+      "grad_norm": 0.5139872431755066,
+      "learning_rate": 2e-05,
+      "loss": 1.131,
+      "step": 1470
+    },
+    {
+      "epoch": 0.1252857021924998,
+      "grad_norm": 0.5229331851005554,
+      "learning_rate": 2e-05,
+      "loss": 1.1198,
+      "step": 1480
+    },
+    {
+      "epoch": 0.12613222720731398,
+      "grad_norm": 0.5084171295166016,
+      "learning_rate": 2e-05,
+      "loss": 1.1001,
+      "step": 1490
+    },
+    {
+      "epoch": 0.12697875222212818,
+      "grad_norm": 0.5192288160324097,
+      "learning_rate": 2e-05,
+      "loss": 1.0706,
+      "step": 1500
+    },
+    {
+      "epoch": 0.12782527723694234,
+      "grad_norm": 0.5821983814239502,
+      "learning_rate": 2e-05,
+      "loss": 1.1429,
+      "step": 1510
+    },
+    {
+      "epoch": 0.12867180225175653,
+      "grad_norm": 0.5315274596214294,
+      "learning_rate": 2e-05,
+      "loss": 1.119,
+      "step": 1520
+    },
+    {
+      "epoch": 0.12951832726657073,
+      "grad_norm": 0.5352998971939087,
+      "learning_rate": 2e-05,
+      "loss": 1.145,
+      "step": 1530
+    },
+    {
+      "epoch": 0.13036485228138492,
+      "grad_norm": 0.5305722951889038,
+      "learning_rate": 2e-05,
+      "loss": 1.1264,
+      "step": 1540
+    },
+    {
+      "epoch": 0.1312113772961991,
+      "grad_norm": 0.5620173215866089,
+      "learning_rate": 2e-05,
+      "loss": 1.1474,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1320579023110133,
+      "grad_norm": 0.512162446975708,
+      "learning_rate": 2e-05,
+      "loss": 1.1036,
+      "step": 1560
+    },
+    {
+      "epoch": 0.13290442732582747,
+      "grad_norm": 0.5346412658691406,
+      "learning_rate": 2e-05,
+      "loss": 1.1447,
+      "step": 1570
+    },
+    {
+      "epoch": 0.13375095234064166,
+      "grad_norm": 0.5146453976631165,
+      "learning_rate": 2e-05,
+      "loss": 1.1617,
+      "step": 1580
+    },
+    {
+      "epoch": 0.13459747735545585,
+      "grad_norm": 0.5036106109619141,
+      "learning_rate": 2e-05,
+      "loss": 1.124,
+      "step": 1590
+    },
+    {
+      "epoch": 0.13544400237027004,
+      "grad_norm": 0.5062596201896667,
+      "learning_rate": 2e-05,
+      "loss": 1.164,
+      "step": 1600
+    },
+    {
+      "epoch": 0.13629052738508424,
+      "grad_norm": 0.5168201327323914,
+      "learning_rate": 2e-05,
+      "loss": 1.1139,
+      "step": 1610
+    },
+    {
+      "epoch": 0.13713705239989843,
+      "grad_norm": 0.49784719944000244,
+      "learning_rate": 2e-05,
+      "loss": 1.1462,
+      "step": 1620
+    },
+    {
+      "epoch": 0.1379835774147126,
+      "grad_norm": 0.5189639925956726,
+      "learning_rate": 2e-05,
+      "loss": 1.1086,
+      "step": 1630
+    },
+    {
+      "epoch": 0.13883010242952679,
+      "grad_norm": 0.506519079208374,
+      "learning_rate": 2e-05,
+      "loss": 1.1325,
+      "step": 1640
+    },
+    {
+      "epoch": 0.13967662744434098,
+      "grad_norm": 0.5217747688293457,
+      "learning_rate": 2e-05,
+      "loss": 1.0839,
+      "step": 1650
+    },
+    {
+      "epoch": 0.14052315245915517,
+      "grad_norm": 0.5097542405128479,
+      "learning_rate": 2e-05,
+      "loss": 1.1274,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14136967747396936,
+      "grad_norm": 0.5042250156402588,
+      "learning_rate": 2e-05,
+      "loss": 1.1181,
+      "step": 1670
+    },
+    {
+      "epoch": 0.14221620248878356,
+      "grad_norm": 0.5155004858970642,
+      "learning_rate": 2e-05,
+      "loss": 1.117,
+      "step": 1680
+    },
+    {
+      "epoch": 0.14306272750359772,
+      "grad_norm": 0.5247176885604858,
+      "learning_rate": 2e-05,
+      "loss": 1.1446,
+      "step": 1690
+    },
+    {
+      "epoch": 0.1439092525184119,
+      "grad_norm": 0.5262718796730042,
+      "learning_rate": 2e-05,
+      "loss": 1.123,
+      "step": 1700
+    },
+    {
+      "epoch": 0.1447557775332261,
+      "grad_norm": 0.5335429906845093,
+      "learning_rate": 2e-05,
+      "loss": 1.0919,
+      "step": 1710
+    },
+    {
+      "epoch": 0.1456023025480403,
+      "grad_norm": 0.50660640001297,
+      "learning_rate": 2e-05,
+      "loss": 1.1377,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1464488275628545,
+      "grad_norm": 0.5110899209976196,
+      "learning_rate": 2e-05,
+      "loss": 1.1272,
+      "step": 1730
+    },
+    {
+      "epoch": 0.14729535257766868,
+      "grad_norm": 0.5016074776649475,
+      "learning_rate": 2e-05,
+      "loss": 1.1189,
+      "step": 1740
+    },
+    {
+      "epoch": 0.14814187759248285,
+      "grad_norm": 0.5078844428062439,
+      "learning_rate": 2e-05,
+      "loss": 1.1286,
+      "step": 1750
+    },
+    {
+      "epoch": 0.14898840260729704,
+      "grad_norm": 0.530308187007904,
+      "learning_rate": 2e-05,
+      "loss": 1.1392,
+      "step": 1760
+    },
+    {
+      "epoch": 0.14983492762211123,
+      "grad_norm": 0.5045443773269653,
+      "learning_rate": 2e-05,
+      "loss": 1.1255,
+      "step": 1770
+    },
+    {
+      "epoch": 0.15068145263692542,
+      "grad_norm": 0.5475932359695435,
+      "learning_rate": 2e-05,
+      "loss": 1.1192,
+      "step": 1780
+    },
+    {
+      "epoch": 0.15152797765173961,
+      "grad_norm": 0.5143741369247437,
+      "learning_rate": 2e-05,
+      "loss": 1.1135,
+      "step": 1790
+    },
+    {
+      "epoch": 0.1523745026665538,
+      "grad_norm": 0.5300629734992981,
+      "learning_rate": 2e-05,
+      "loss": 1.122,
+      "step": 1800
+    },
+    {
+      "epoch": 0.15322102768136797,
+      "grad_norm": 0.5044997334480286,
+      "learning_rate": 2e-05,
+      "loss": 1.1239,
+      "step": 1810
+    },
+    {
+      "epoch": 0.15406755269618216,
+      "grad_norm": 0.5133723020553589,
+      "learning_rate": 2e-05,
+      "loss": 1.1383,
+      "step": 1820
+    },
+    {
+      "epoch": 0.15491407771099636,
+      "grad_norm": 0.5047634840011597,
+      "learning_rate": 2e-05,
+      "loss": 1.1241,
+      "step": 1830
+    },
+    {
+      "epoch": 0.15576060272581055,
+      "grad_norm": 0.5347635746002197,
+      "learning_rate": 2e-05,
+      "loss": 1.1229,
+      "step": 1840
+    },
+    {
+      "epoch": 0.15660712774062474,
+      "grad_norm": 0.5044793486595154,
+      "learning_rate": 2e-05,
+      "loss": 1.1054,
+      "step": 1850
+    },
+    {
+      "epoch": 0.15745365275543893,
+      "grad_norm": 0.4948312044143677,
+      "learning_rate": 2e-05,
+      "loss": 1.1024,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1583001777702531,
+      "grad_norm": 0.5021107792854309,
+      "learning_rate": 2e-05,
+      "loss": 1.1289,
+      "step": 1870
+    },
+    {
+      "epoch": 0.1591467027850673,
+      "grad_norm": 0.5103649497032166,
+      "learning_rate": 2e-05,
+      "loss": 1.126,
+      "step": 1880
+    },
+    {
+      "epoch": 0.15999322779988148,
+      "grad_norm": 0.5388750433921814,
+      "learning_rate": 2e-05,
+      "loss": 1.0896,
+      "step": 1890
+    },
+    {
+      "epoch": 0.16083975281469567,
+      "grad_norm": 0.5113123059272766,
+      "learning_rate": 2e-05,
+      "loss": 1.1139,
+      "step": 1900
+    },
+    {
+      "epoch": 0.16168627782950987,
+      "grad_norm": 0.5025059580802917,
+      "learning_rate": 2e-05,
+      "loss": 1.1129,
+      "step": 1910
+    },
+    {
+      "epoch": 0.16253280284432406,
+      "grad_norm": 0.5404525995254517,
+      "learning_rate": 2e-05,
+      "loss": 1.1123,
+      "step": 1920
+    },
+    {
+      "epoch": 0.16337932785913825,
+      "grad_norm": 0.529439389705658,
+      "learning_rate": 2e-05,
+      "loss": 1.131,
+      "step": 1930
+    },
+    {
+      "epoch": 0.16422585287395242,
+      "grad_norm": 0.5168623328208923,
+      "learning_rate": 2e-05,
+      "loss": 1.1224,
+      "step": 1940
+    },
+    {
+      "epoch": 0.1650723778887666,
+      "grad_norm": 0.5329291820526123,
+      "learning_rate": 2e-05,
+      "loss": 1.1362,
+      "step": 1950
+    },
+    {
+      "epoch": 0.1659189029035808,
+      "grad_norm": 0.4997026026248932,
+      "learning_rate": 2e-05,
+      "loss": 1.1231,
+      "step": 1960
+    },
+    {
+      "epoch": 0.166765427918395,
+      "grad_norm": 0.5062265396118164,
+      "learning_rate": 2e-05,
+      "loss": 1.1167,
+      "step": 1970
+    },
+    {
+      "epoch": 0.16761195293320919,
+      "grad_norm": 0.5036337375640869,
+      "learning_rate": 2e-05,
+      "loss": 1.1348,
+      "step": 1980
+    },
+    {
+      "epoch": 0.16845847794802338,
+      "grad_norm": 0.5178899168968201,
+      "learning_rate": 2e-05,
+      "loss": 1.0872,
+      "step": 1990
+    },
+    {
+      "epoch": 0.16930500296283754,
+      "grad_norm": 0.5466113090515137,
+      "learning_rate": 2e-05,
+      "loss": 1.1262,
+      "step": 2000
+    },
+    {
+      "epoch": 0.17015152797765173,
+      "grad_norm": 0.5311413407325745,
+      "learning_rate": 2e-05,
+      "loss": 1.0855,
+      "step": 2010
+    },
+    {
+      "epoch": 0.17099805299246593,
+      "grad_norm": 0.5100221633911133,
+      "learning_rate": 2e-05,
+      "loss": 1.1309,
+      "step": 2020
+    },
+    {
+      "epoch": 0.17184457800728012,
+      "grad_norm": 0.5275105834007263,
+      "learning_rate": 2e-05,
+      "loss": 1.0994,
+      "step": 2030
+    },
+    {
+      "epoch": 0.1726911030220943,
+      "grad_norm": 0.5669128894805908,
+      "learning_rate": 2e-05,
+      "loss": 1.1114,
+      "step": 2040
+    },
+    {
+      "epoch": 0.1735376280369085,
+      "grad_norm": 0.5159915089607239,
+      "learning_rate": 2e-05,
+      "loss": 1.1099,
+      "step": 2050
+    },
+    {
+      "epoch": 0.17438415305172267,
+      "grad_norm": 0.4939943850040436,
+      "learning_rate": 2e-05,
+      "loss": 1.1168,
+      "step": 2060
+    },
+    {
+      "epoch": 0.17523067806653686,
+      "grad_norm": 0.4930158257484436,
+      "learning_rate": 2e-05,
+      "loss": 1.094,
+      "step": 2070
+    },
+    {
+      "epoch": 0.17607720308135105,
+      "grad_norm": 0.4986574649810791,
+      "learning_rate": 2e-05,
+      "loss": 1.1146,
+      "step": 2080
+    },
+    {
+      "epoch": 0.17692372809616524,
+      "grad_norm": 0.5078601241111755,
+      "learning_rate": 2e-05,
+      "loss": 1.1148,
+      "step": 2090
+    },
+    {
+      "epoch": 0.17777025311097944,
+      "grad_norm": 0.4965764880180359,
+      "learning_rate": 2e-05,
+      "loss": 1.0842,
+      "step": 2100
+    },
+    {
+      "epoch": 0.17861677812579363,
+      "grad_norm": 0.5152555108070374,
+      "learning_rate": 2e-05,
+      "loss": 1.1155,
+      "step": 2110
+    },
+    {
+      "epoch": 0.1794633031406078,
+      "grad_norm": 0.5084643959999084,
+      "learning_rate": 2e-05,
+      "loss": 1.0979,
+      "step": 2120
+    },
+    {
+      "epoch": 0.180309828155422,
+      "grad_norm": 0.51066654920578,
+      "learning_rate": 2e-05,
+      "loss": 1.1286,
+      "step": 2130
+    },
+    {
+      "epoch": 0.18115635317023618,
+      "grad_norm": 0.527068018913269,
+      "learning_rate": 2e-05,
+      "loss": 1.1208,
+      "step": 2140
+    },
+    {
+      "epoch": 0.18200287818505037,
+      "grad_norm": 0.4924015700817108,
+      "learning_rate": 2e-05,
+      "loss": 1.122,
+      "step": 2150
+    },
+    {
+      "epoch": 0.18284940319986456,
+      "grad_norm": 0.5086539387702942,
+      "learning_rate": 2e-05,
+      "loss": 1.0952,
+      "step": 2160
+    },
+    {
+      "epoch": 0.18369592821467876,
+      "grad_norm": 0.5031940937042236,
+      "learning_rate": 2e-05,
+      "loss": 1.1318,
+      "step": 2170
+    },
+    {
+      "epoch": 0.18454245322949292,
+      "grad_norm": 0.5074408054351807,
+      "learning_rate": 2e-05,
+      "loss": 1.1128,
+      "step": 2180
+    },
+    {
+      "epoch": 0.1853889782443071,
+      "grad_norm": 0.48353567719459534,
+      "learning_rate": 2e-05,
+      "loss": 1.0799,
+      "step": 2190
+    },
+    {
+      "epoch": 0.1862355032591213,
+      "grad_norm": 0.5043293237686157,
+      "learning_rate": 2e-05,
+      "loss": 1.0976,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1870820282739355,
+      "grad_norm": 0.5117511749267578,
+      "learning_rate": 2e-05,
+      "loss": 1.1011,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1879285532887497,
+      "grad_norm": 0.512195348739624,
+      "learning_rate": 2e-05,
+      "loss": 1.1143,
+      "step": 2220
+    },
+    {
+      "epoch": 0.18877507830356388,
+      "grad_norm": 0.4908454120159149,
+      "learning_rate": 2e-05,
+      "loss": 1.1018,
+      "step": 2230
+    },
+    {
+      "epoch": 0.18962160331837805,
+      "grad_norm": 0.504994809627533,
+      "learning_rate": 2e-05,
+      "loss": 1.1229,
+      "step": 2240
+    },
+    {
+      "epoch": 0.19046812833319224,
+      "grad_norm": 0.5080342292785645,
+      "learning_rate": 2e-05,
+      "loss": 1.1015,
+      "step": 2250
+    },
+    {
+      "epoch": 0.19131465334800643,
+      "grad_norm": 0.5111969113349915,
+      "learning_rate": 2e-05,
+      "loss": 1.1104,
+      "step": 2260
+    },
+    {
+      "epoch": 0.19216117836282062,
+      "grad_norm": 0.5021278858184814,
+      "learning_rate": 2e-05,
+      "loss": 1.0976,
+      "step": 2270
+    },
+    {
+      "epoch": 0.19300770337763481,
+      "grad_norm": 0.7183691263198853,
+      "learning_rate": 2e-05,
+      "loss": 1.0984,
+      "step": 2280
+    },
+    {
+      "epoch": 0.193854228392449,
+      "grad_norm": 0.5097667574882507,
+      "learning_rate": 2e-05,
+      "loss": 1.1205,
+      "step": 2290
+    },
+    {
+      "epoch": 0.19470075340726317,
+      "grad_norm": 0.5166229605674744,
+      "learning_rate": 2e-05,
+      "loss": 1.0896,
+      "step": 2300
+    },
+    {
+      "epoch": 0.19554727842207736,
+      "grad_norm": 0.49435800313949585,
+      "learning_rate": 2e-05,
+      "loss": 1.0734,
+      "step": 2310
+    },
+    {
+      "epoch": 0.19639380343689156,
+      "grad_norm": 0.5085446238517761,
+      "learning_rate": 2e-05,
+      "loss": 1.1253,
+      "step": 2320
+    },
+    {
+      "epoch": 0.19724032845170575,
+      "grad_norm": 0.4996432363986969,
+      "learning_rate": 2e-05,
+      "loss": 1.0908,
+      "step": 2330
+    },
+    {
+      "epoch": 0.19808685346651994,
+      "grad_norm": 0.5074278712272644,
+      "learning_rate": 2e-05,
+      "loss": 1.1153,
+      "step": 2340
+    },
+    {
+      "epoch": 0.19893337848133413,
+      "grad_norm": 0.5085509419441223,
+      "learning_rate": 2e-05,
+      "loss": 1.1132,
+      "step": 2350
+    },
+    {
+      "epoch": 0.1997799034961483,
+      "grad_norm": 0.49525755643844604,
+      "learning_rate": 2e-05,
+      "loss": 1.1016,
+      "step": 2360
+    },
+    {
+      "epoch": 0.2006264285109625,
+      "grad_norm": 0.49528446793556213,
+      "learning_rate": 2e-05,
+      "loss": 1.1306,
+      "step": 2370
+    },
+    {
+      "epoch": 0.20147295352577668,
+      "grad_norm": 0.4936249256134033,
+      "learning_rate": 2e-05,
+      "loss": 1.0911,
+      "step": 2380
+    },
+    {
+      "epoch": 0.20231947854059087,
+      "grad_norm": 0.4922093152999878,
+      "learning_rate": 2e-05,
+      "loss": 1.0979,
+      "step": 2390
+    },
+    {
+      "epoch": 0.20316600355540507,
+      "grad_norm": 0.5058261752128601,
+      "learning_rate": 2e-05,
+      "loss": 1.1113,
+      "step": 2400
+    },
+    {
+      "epoch": 0.20401252857021926,
+      "grad_norm": 0.49868348240852356,
+      "learning_rate": 2e-05,
+      "loss": 1.0985,
+      "step": 2410
+    },
+    {
+      "epoch": 0.20485905358503345,
+      "grad_norm": 0.4947226047515869,
+      "learning_rate": 2e-05,
+      "loss": 1.117,
+      "step": 2420
+    },
+    {
+      "epoch": 0.20570557859984762,
+      "grad_norm": 0.5019698143005371,
+      "learning_rate": 2e-05,
+      "loss": 1.1138,
+      "step": 2430
+    },
+    {
+      "epoch": 0.2065521036146618,
+      "grad_norm": 0.4983386695384979,
+      "learning_rate": 2e-05,
+      "loss": 1.1025,
+      "step": 2440
+    },
+    {
+      "epoch": 0.207398628629476,
+      "grad_norm": 0.4934694170951843,
+      "learning_rate": 2e-05,
+      "loss": 1.0966,
+      "step": 2450
+    },
+    {
+      "epoch": 0.2082451536442902,
+      "grad_norm": 0.4954778254032135,
+      "learning_rate": 2e-05,
+      "loss": 1.1139,
+      "step": 2460
+    },
+    {
+      "epoch": 0.20909167865910439,
+      "grad_norm": 0.4950241148471832,
+      "learning_rate": 2e-05,
+      "loss": 1.1136,
+      "step": 2470
+    },
+    {
+      "epoch": 0.20993820367391858,
+      "grad_norm": 0.5022818446159363,
+      "learning_rate": 2e-05,
+      "loss": 1.1013,
+      "step": 2480
+    },
+    {
+      "epoch": 0.21078472868873274,
+      "grad_norm": 0.496825248003006,
+      "learning_rate": 2e-05,
+      "loss": 1.0958,
+      "step": 2490
+    },
+    {
+      "epoch": 0.21163125370354693,
+      "grad_norm": 0.48725008964538574,
+      "learning_rate": 2e-05,
+      "loss": 1.1046,
+      "step": 2500
+    },
+    {
+      "epoch": 0.21247777871836113,
+      "grad_norm": 0.5036442279815674,
+      "learning_rate": 2e-05,
+      "loss": 1.0948,
+      "step": 2510
+    },
+    {
+      "epoch": 0.21332430373317532,
+      "grad_norm": 0.5268523097038269,
+      "learning_rate": 2e-05,
+      "loss": 1.1061,
+      "step": 2520
+    },
+    {
+      "epoch": 0.2141708287479895,
+      "grad_norm": 0.4953404664993286,
+      "learning_rate": 2e-05,
+      "loss": 1.0826,
+      "step": 2530
+    },
+    {
+      "epoch": 0.2150173537628037,
+      "grad_norm": 0.4943073093891144,
+      "learning_rate": 2e-05,
+      "loss": 1.0876,
+      "step": 2540
+    },
+    {
+      "epoch": 0.21586387877761787,
+      "grad_norm": 0.5178449153900146,
+      "learning_rate": 1.9980979278401095e-05,
+      "loss": 1.0796,
+      "step": 2550
+    },
+    {
+      "epoch": 0.21671040379243206,
+      "grad_norm": 0.49626877903938293,
+      "learning_rate": 1.982924955463327e-05,
+      "loss": 1.1063,
+      "step": 2560
+    },
+    {
+      "epoch": 0.21755692880724625,
+      "grad_norm": 0.49790918827056885,
+      "learning_rate": 1.9528108298328235e-05,
+      "loss": 1.0726,
+      "step": 2570
+    },
+    {
+      "epoch": 0.21840345382206044,
+      "grad_norm": 0.5012426376342773,
+      "learning_rate": 1.9082156473635994e-05,
+      "loss": 1.1088,
+      "step": 2580
+    },
+    {
+      "epoch": 0.21924997883687464,
+      "grad_norm": 0.5069933533668518,
+      "learning_rate": 1.8498207522139434e-05,
+      "loss": 1.1222,
+      "step": 2590
+    },
+    {
+      "epoch": 0.22009650385168883,
+      "grad_norm": 0.4929569661617279,
+      "learning_rate": 1.7785183264196977e-05,
+      "loss": 1.1029,
+      "step": 2600
+    },
+    {
+      "epoch": 0.220943028866503,
+      "grad_norm": 0.4802176058292389,
+      "learning_rate": 1.6953977587577846e-05,
+      "loss": 1.1041,
+      "step": 2610
+    },
+    {
+      "epoch": 0.2217895538813172,
+      "grad_norm": 0.5004302263259888,
+      "learning_rate": 1.601729000601297e-05,
+      "loss": 1.0977,
+      "step": 2620
+    },
+    {
+      "epoch": 0.22263607889613138,
+      "grad_norm": 0.5074357390403748,
+      "learning_rate": 1.4989431630624763e-05,
+      "loss": 1.109,
+      "step": 2630
+    },
+    {
+      "epoch": 0.22348260391094557,
+      "grad_norm": 0.48369473218917847,
+      "learning_rate": 1.3886106518686746e-05,
+      "loss": 1.1031,
+      "step": 2640
+    },
+    {
+      "epoch": 0.22432912892575976,
+      "grad_norm": 0.4933447539806366,
+      "learning_rate": 1.2724171740359612e-05,
+      "loss": 1.0749,
+      "step": 2650
+    },
+    {
+      "epoch": 0.22517565394057396,
+      "grad_norm": 0.5434425473213196,
+      "learning_rate": 1.1521379829206218e-05,
+      "loss": 1.0914,
+      "step": 2660
+    },
+    {
+      "epoch": 0.22602217895538812,
+      "grad_norm": 0.48402291536331177,
+      "learning_rate": 1.029610755143595e-05,
+      "loss": 1.1083,
+      "step": 2670
+    },
+    {
+      "epoch": 0.2268687039702023,
+      "grad_norm": 0.48168373107910156,
+      "learning_rate": 9.067075137857348e-06,
+      "loss": 1.0708,
+      "step": 2680
+    },
+    {
+      "epoch": 0.2277152289850165,
+      "grad_norm": 0.4888278841972351,
+      "learning_rate": 7.853060268232525e-06,
+      "loss": 1.0812,
+      "step": 2690
+    },
+    {
+      "epoch": 0.2285617539998307,
+      "grad_norm": 0.4868822991847992,
+      "learning_rate": 6.672611177902054e-06,
+      "loss": 1.0822,
+      "step": 2700
+    },
+    {
+      "epoch": 0.2294082790146449,
+      "grad_norm": 0.4879697263240814,
+      "learning_rate": 5.543763269959312e-06,
+      "loss": 1.0845,
+      "step": 2710
+    },
+    {
+      "epoch": 0.23025480402945908,
+      "grad_norm": 0.48054879903793335,
+      "learning_rate": 4.483763562693867e-06,
+      "loss": 1.0787,
+      "step": 2720
+    },
+    {
+      "epoch": 0.23110132904427325,
+      "grad_norm": 0.47858574986457825,
+      "learning_rate": 3.508807182313187e-06,
+      "loss": 1.086,
+      "step": 2730
+    },
+    {
+      "epoch": 0.23194785405908744,
+      "grad_norm": 0.48914384841918945,
+      "learning_rate": 2.6337899269187294e-06,
+      "loss": 1.0848,
+      "step": 2740
+    },
+    {
+      "epoch": 0.23279437907390163,
+      "grad_norm": 0.47290483117103577,
+      "learning_rate": 1.8720806821693126e-06,
+      "loss": 1.0855,
+      "step": 2750
+    },
+    {
+      "epoch": 0.23364090408871582,
+      "grad_norm": 0.46513378620147705,
+      "learning_rate": 1.2353171657619887e-06,
+      "loss": 1.0752,
+      "step": 2760
+    },
+    {
+      "epoch": 0.23448742910353002,
+      "grad_norm": 0.4818378984928131,
+      "learning_rate": 7.332281214330356e-07,
+      "loss": 1.0848,
+      "step": 2770
+    },
+    {
+      "epoch": 0.2353339541183442,
+      "grad_norm": 0.4728337824344635,
+      "learning_rate": 3.73484679074808e-07,
+      "loss": 1.0935,
+      "step": 2780
+    },
+    {
+      "epoch": 0.23618047913315837,
+      "grad_norm": 0.49928024411201477,
+      "learning_rate": 1.61583151951702e-07,
+      "loss": 1.0823,
+      "step": 2790
+    },
+    {
+      "epoch": 0.23685769914500973,
+      "step": 2798,
+      "total_flos": 1.1366756130249245e+19,
+      "train_loss": 0.09888922034884624,
+      "train_runtime": 2290.2876,
+      "train_samples_per_second": 14.66,
+      "train_steps_per_second": 1.222
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2798,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1366756130249245e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g1_blog_post__edit/500M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.4738000507915009,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.03504324709067754,
+    "train_runtime": 1761.3031,
+    "train_samples_per_second": 38.133,
+    "train_steps_per_second": 3.178
+}

g1_blog_post__edit/500M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_blog_post__edit/500M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_blog_post__edit/500M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_blog_post__edit/500M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_blog_post__edit/500M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}