Add files using upload-large-folder tool

g1_0_baseline_original_code/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0049376066074154,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.0036866624205798702,
+    "train_runtime": 610.6134,
+    "train_samples_per_second": 219.989,
+    "train_steps_per_second": 18.332
+}

g1_0_baseline_original_code/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_0_baseline_original_code/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_0_baseline_original_code/1000M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_0_baseline_original_code/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_0_baseline_original_code/1000M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_0_baseline_original_code/1000M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_0_baseline_original_code/1000M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0049376066074154,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.0036866624205798702,
+    "train_runtime": 610.6134,
+    "train_samples_per_second": 219.989,
+    "train_steps_per_second": 18.332
+}

g1_0_baseline_original_code/1000M/trainer_log.jsonl

@@ -0,0 +1,123 @@
+{"current_steps": 10180, "total_steps": 11194, "loss": 0.794, "lr": 1.999957358086795e-05, "epoch": 0.9139060956997935, "percentage": 90.94, "elapsed_time": "0:00:38", "remaining_time": "0:00:03"}
+{"current_steps": 10190, "total_steps": 11194, "loss": 0.7012, "lr": 1.9991993813020483e-05, "epoch": 0.9148038423556872, "percentage": 91.03, "elapsed_time": "0:02:07", "remaining_time": "0:00:12"}
+{"current_steps": 10200, "total_steps": 11194, "loss": 0.7632, "lr": 1.9974946373028628e-05, "epoch": 0.9157015890115809, "percentage": 91.12, "elapsed_time": "0:03:37", "remaining_time": "0:00:21"}
+{"current_steps": 10210, "total_steps": 11194, "loss": 0.696, "lr": 1.994844749501764e-05, "epoch": 0.9165993356674746, "percentage": 91.21, "elapsed_time": "0:05:06", "remaining_time": "0:00:29"}
+{"current_steps": 10220, "total_steps": 11194, "loss": 0.7021, "lr": 1.9912522413632988e-05, "epoch": 0.9174970823233684, "percentage": 91.3, "elapsed_time": "0:06:36", "remaining_time": "0:00:37"}
+{"current_steps": 10230, "total_steps": 11194, "loss": 0.7559, "lr": 1.9867205340009656e-05, "epoch": 0.9183948289792621, "percentage": 91.39, "elapsed_time": "0:08:05", "remaining_time": "0:00:45"}
+{"current_steps": 10240, "total_steps": 11194, "loss": 0.7967, "lr": 1.9812539429193164e-05, "epoch": 0.9192925756351558, "percentage": 91.48, "elapsed_time": "0:09:35", "remaining_time": "0:00:53"}
+{"current_steps": 10250, "total_steps": 11194, "loss": 0.7048, "lr": 1.9748576739043423e-05, "epoch": 0.9201903222910495, "percentage": 91.57, "elapsed_time": "0:11:04", "remaining_time": "0:01:01"}
+{"current_steps": 10260, "total_steps": 11194, "loss": 0.7856, "lr": 1.967537818066047e-05, "epoch": 0.9210880689469432, "percentage": 91.66, "elapsed_time": "0:12:34", "remaining_time": "0:01:08"}
+{"current_steps": 10270, "total_steps": 11194, "loss": 0.7683, "lr": 1.959301346037941e-05, "epoch": 0.9219858156028369, "percentage": 91.75, "elapsed_time": "0:14:03", "remaining_time": "0:01:15"}
+{"current_steps": 10280, "total_steps": 11194, "loss": 0.7607, "lr": 1.9501561013389646e-05, "epoch": 0.9228835622587306, "percentage": 91.83, "elapsed_time": "0:15:33", "remaining_time": "0:01:22"}
+{"current_steps": 10290, "total_steps": 11194, "loss": 0.7159, "lr": 1.9401107929041795e-05, "epoch": 0.9237813089146243, "percentage": 91.92, "elapsed_time": "0:17:02", "remaining_time": "0:01:29"}
+{"current_steps": 10300, "total_steps": 11194, "loss": 0.7054, "lr": 1.9291749867913225e-05, "epoch": 0.924679055570518, "percentage": 92.01, "elapsed_time": "0:18:32", "remaining_time": "0:01:36"}
+{"current_steps": 10180, "total_steps": 11194, "loss": 0.794, "lr": 1.999957358086795e-05, "epoch": 0.9139060956997935, "percentage": 90.94, "elapsed_time": "0:00:37", "remaining_time": "0:00:03"}
+{"current_steps": 10190, "total_steps": 11194, "loss": 0.701, "lr": 1.9991993813020483e-05, "epoch": 0.9148038423556872, "percentage": 91.03, "elapsed_time": "0:02:05", "remaining_time": "0:00:12"}
+{"current_steps": 10200, "total_steps": 11194, "loss": 0.7632, "lr": 1.9974946373028628e-05, "epoch": 0.9157015890115809, "percentage": 91.12, "elapsed_time": "0:03:34", "remaining_time": "0:00:20"}
+{"current_steps": 10210, "total_steps": 11194, "loss": 0.696, "lr": 1.994844749501764e-05, "epoch": 0.9165993356674746, "percentage": 91.21, "elapsed_time": "0:05:02", "remaining_time": "0:00:29"}
+{"current_steps": 10220, "total_steps": 11194, "loss": 0.7021, "lr": 1.9912522413632988e-05, "epoch": 0.9174970823233684, "percentage": 91.3, "elapsed_time": "0:06:30", "remaining_time": "0:00:37"}
+{"current_steps": 10230, "total_steps": 11194, "loss": 0.7559, "lr": 1.9867205340009656e-05, "epoch": 0.9183948289792621, "percentage": 91.39, "elapsed_time": "0:07:58", "remaining_time": "0:00:45"}
+{"current_steps": 10240, "total_steps": 11194, "loss": 0.7966, "lr": 1.9812539429193164e-05, "epoch": 0.9192925756351558, "percentage": 91.48, "elapsed_time": "0:09:26", "remaining_time": "0:00:52"}
+{"current_steps": 10250, "total_steps": 11194, "loss": 0.705, "lr": 1.9748576739043423e-05, "epoch": 0.9201903222910495, "percentage": 91.57, "elapsed_time": "0:10:54", "remaining_time": "0:01:00"}
+{"current_steps": 10260, "total_steps": 11194, "loss": 0.7857, "lr": 1.967537818066047e-05, "epoch": 0.9210880689469432, "percentage": 91.66, "elapsed_time": "0:12:22", "remaining_time": "0:01:07"}
+{"current_steps": 10270, "total_steps": 11194, "loss": 0.7684, "lr": 1.959301346037941e-05, "epoch": 0.9219858156028369, "percentage": 91.75, "elapsed_time": "0:13:51", "remaining_time": "0:01:14"}
+{"current_steps": 10280, "total_steps": 11194, "loss": 0.7608, "lr": 1.9501561013389646e-05, "epoch": 0.9228835622587306, "percentage": 91.83, "elapsed_time": "0:15:19", "remaining_time": "0:01:21"}
+{"current_steps": 10290, "total_steps": 11194, "loss": 0.7158, "lr": 1.9401107929041795e-05, "epoch": 0.9237813089146243, "percentage": 91.92, "elapsed_time": "0:16:47", "remaining_time": "0:01:28"}
+{"current_steps": 10300, "total_steps": 11194, "loss": 0.7055, "lr": 1.9291749867913225e-05, "epoch": 0.924679055570518, "percentage": 92.01, "elapsed_time": "0:18:15", "remaining_time": "0:01:35"}
+{"current_steps": 10310, "total_steps": 11194, "loss": 0.7618, "lr": 1.91735909707114e-05, "epoch": 0.9255768022264117, "percentage": 92.1, "elapsed_time": "0:19:43", "remaining_time": "0:01:41"}
+{"current_steps": 10320, "total_steps": 11194, "loss": 0.771, "lr": 1.9046743759101545e-05, "epoch": 0.9264745488823054, "percentage": 92.19, "elapsed_time": "0:21:12", "remaining_time": "0:01:47"}
+{"current_steps": 10330, "total_steps": 11194, "loss": 0.7254, "lr": 1.8911329028553324e-05, "epoch": 0.9273722955381991, "percentage": 92.28, "elapsed_time": "0:22:40", "remaining_time": "0:01:53"}
+{"current_steps": 10340, "total_steps": 11194, "loss": 0.7772, "lr": 1.876747573330835e-05, "epoch": 0.9282700421940928, "percentage": 92.37, "elapsed_time": "0:24:46", "remaining_time": "0:02:02"}
+{"current_steps": 10350, "total_steps": 11194, "loss": 0.7788, "lr": 1.8615320863578244e-05, "epoch": 0.9291677888499865, "percentage": 92.46, "elapsed_time": "0:26:14", "remaining_time": "0:02:08"}
+{"current_steps": 10360, "total_steps": 11194, "loss": 0.7634, "lr": 1.8455009315090036e-05, "epoch": 0.9300655355058802, "percentage": 92.55, "elapsed_time": "0:27:42", "remaining_time": "0:02:13"}
+{"current_steps": 10370, "total_steps": 11194, "loss": 0.7756, "lr": 1.8286693751103294e-05, "epoch": 0.9309632821617739, "percentage": 92.64, "elapsed_time": "0:29:10", "remaining_time": "0:02:19"}
+{"current_steps": 10380, "total_steps": 11194, "loss": 0.7936, "lr": 1.811053445703024e-05, "epoch": 0.9318610288176676, "percentage": 92.73, "elapsed_time": "0:30:38", "remaining_time": "0:02:24"}
+{"current_steps": 10390, "total_steps": 11194, "loss": 0.7347, "lr": 1.7926699187797375e-05, "epoch": 0.9327587754735613, "percentage": 92.82, "elapsed_time": "0:32:07", "remaining_time": "0:02:29"}
+{"current_steps": 10400, "total_steps": 11194, "loss": 0.7375, "lr": 1.7735363008093995e-05, "epoch": 0.933656522129455, "percentage": 92.91, "elapsed_time": "0:33:35", "remaining_time": "0:02:33"}
+{"current_steps": 10410, "total_steps": 11194, "loss": 0.7652, "lr": 1.7536708125659577e-05, "epoch": 0.9345542687853488, "percentage": 93.0, "elapsed_time": "0:35:03", "remaining_time": "0:02:38"}
+{"current_steps": 10420, "total_steps": 11194, "loss": 0.7188, "lr": 1.733092371776905e-05, "epoch": 0.9354520154412425, "percentage": 93.09, "elapsed_time": "0:36:31", "remaining_time": "0:02:42"}
+{"current_steps": 10430, "total_steps": 11194, "loss": 0.8431, "lr": 1.7118205751080915e-05, "epoch": 0.9363497620971362, "percentage": 93.17, "elapsed_time": "0:37:59", "remaining_time": "0:02:46"}
+{"current_steps": 10440, "total_steps": 11194, "loss": 0.729, "lr": 1.6898756795019935e-05, "epoch": 0.9372475087530299, "percentage": 93.26, "elapsed_time": "0:39:27", "remaining_time": "0:02:51"}
+{"current_steps": 10450, "total_steps": 11194, "loss": 0.788, "lr": 1.667278582887211e-05, "epoch": 0.9381452554089236, "percentage": 93.35, "elapsed_time": "0:40:56", "remaining_time": "0:02:54"}
+{"current_steps": 10460, "total_steps": 11194, "loss": 0.7499, "lr": 1.6440508042775545e-05, "epoch": 0.9390430020648173, "percentage": 93.44, "elapsed_time": "0:42:24", "remaining_time": "0:02:58"}
+{"current_steps": 10470, "total_steps": 11194, "loss": 0.7409, "lr": 1.6202144632796796e-05, "epoch": 0.939940748720711, "percentage": 93.53, "elapsed_time": "0:43:52", "remaining_time": "0:03:02"}
+{"current_steps": 10480, "total_steps": 11194, "loss": 0.717, "lr": 1.595792259028783e-05, "epoch": 0.9408384953766047, "percentage": 93.62, "elapsed_time": "0:45:20", "remaining_time": "0:03:05"}
+{"current_steps": 10490, "total_steps": 11194, "loss": 0.7473, "lr": 1.570807448572419e-05, "epoch": 0.9417362420324984, "percentage": 93.71, "elapsed_time": "0:46:48", "remaining_time": "0:03:08"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.8076, "lr": 1.5452838247230197e-05, "epoch": 0.9426339886883921, "percentage": 93.8, "elapsed_time": "0:48:54", "remaining_time": "0:03:13"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.7021, "lr": 1.5192456934002106e-05, "epoch": 0.9435317353442858, "percentage": 93.89, "elapsed_time": "0:50:22", "remaining_time": "0:03:16"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.7734, "lr": 1.492717850484502e-05, "epoch": 0.9444294820001795, "percentage": 93.98, "elapsed_time": "0:51:50", "remaining_time": "0:03:19"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.7816, "lr": 1.5452838247230197e-05, "epoch": 0.9426339886883921, "percentage": 93.8, "elapsed_time": "0:00:57", "remaining_time": "0:00:03"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.7021, "lr": 1.5192456934002106e-05, "epoch": 0.9435317353442858, "percentage": 93.89, "elapsed_time": "0:02:25", "remaining_time": "0:00:09"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.7734, "lr": 1.492717850484502e-05, "epoch": 0.9444294820001795, "percentage": 93.98, "elapsed_time": "0:03:54", "remaining_time": "0:00:14"}
+{"current_steps": 10530, "total_steps": 11194, "loss": 0.7144, "lr": 1.4657255582043904e-05, "epoch": 0.9453272286560732, "percentage": 94.07, "elapsed_time": "0:05:22", "remaining_time": "0:00:20"}
+{"current_steps": 10540, "total_steps": 11194, "loss": 0.7555, "lr": 1.4382945210793663e-05, "epoch": 0.9462249753119669, "percentage": 94.16, "elapsed_time": "0:06:50", "remaining_time": "0:00:25"}
+{"current_steps": 10550, "total_steps": 11194, "loss": 0.8395, "lr": 1.4104508614417267e-05, "epoch": 0.9471227219678606, "percentage": 94.25, "elapsed_time": "0:08:18", "remaining_time": "0:00:30"}
+{"current_steps": 10560, "total_steps": 11194, "loss": 0.7343, "lr": 1.382221094560512e-05, "epoch": 0.9480204686237543, "percentage": 94.34, "elapsed_time": "0:09:47", "remaining_time": "0:00:35"}
+{"current_steps": 10570, "total_steps": 11194, "loss": 0.7045, "lr": 1.3536321033912506e-05, "epoch": 0.948918215279648, "percentage": 94.43, "elapsed_time": "0:11:15", "remaining_time": "0:00:39"}
+{"current_steps": 10580, "total_steps": 11194, "loss": 0.7194, "lr": 1.3247111129755609e-05, "epoch": 0.9498159619355417, "percentage": 94.51, "elapsed_time": "0:12:44", "remaining_time": "0:00:44"}
+{"current_steps": 10590, "total_steps": 11194, "loss": 0.833, "lr": 1.2954856645149868e-05, "epoch": 0.9507137085914354, "percentage": 94.6, "elapsed_time": "0:14:12", "remaining_time": "0:00:48"}
+{"current_steps": 10600, "total_steps": 11194, "loss": 0.7214, "lr": 1.2659835891437553e-05, "epoch": 0.9516114552473292, "percentage": 94.69, "elapsed_time": "0:15:40", "remaining_time": "0:00:52"}
+{"current_steps": 10610, "total_steps": 11194, "loss": 0.7067, "lr": 1.236232981425436e-05, "epoch": 0.9525092019032229, "percentage": 94.78, "elapsed_time": "0:17:08", "remaining_time": "0:00:56"}
+{"current_steps": 10620, "total_steps": 11194, "loss": 0.7471, "lr": 1.2062621725987402e-05, "epoch": 0.9534069485591166, "percentage": 94.87, "elapsed_time": "0:18:37", "remaining_time": "0:01:00"}
+{"current_steps": 10630, "total_steps": 11194, "loss": 0.7627, "lr": 1.1760997035979318e-05, "epoch": 0.9543046952150104, "percentage": 94.96, "elapsed_time": "0:20:05", "remaining_time": "0:01:03"}
+{"current_steps": 10640, "total_steps": 11194, "loss": 0.6691, "lr": 1.1457742978735514e-05, "epoch": 0.9552024418709041, "percentage": 95.05, "elapsed_time": "0:21:33", "remaining_time": "0:01:07"}
+{"current_steps": 10650, "total_steps": 11194, "loss": 0.7132, "lr": 1.1153148340393298e-05, "epoch": 0.9561001885267978, "percentage": 95.14, "elapsed_time": "0:23:02", "remaining_time": "0:01:10"}
+{"current_steps": 10660, "total_steps": 11194, "loss": 0.7051, "lr": 1.0847503183713424e-05, "epoch": 0.9569979351826915, "percentage": 95.23, "elapsed_time": "0:25:08", "remaining_time": "0:01:15"}
+{"current_steps": 10670, "total_steps": 11194, "loss": 0.6889, "lr": 1.0541098571855896e-05, "epoch": 0.9578956818385852, "percentage": 95.32, "elapsed_time": "0:26:37", "remaining_time": "0:01:18"}
+{"current_steps": 10680, "total_steps": 11194, "loss": 0.765, "lr": 1.0234226291203123e-05, "epoch": 0.9587934284944789, "percentage": 95.41, "elapsed_time": "0:28:05", "remaining_time": "0:01:21"}
+{"current_steps": 10690, "total_steps": 11194, "loss": 0.7654, "lr": 9.927178573494333e-06, "epoch": 0.9596911751503726, "percentage": 95.5, "elapsed_time": "0:29:33", "remaining_time": "0:01:23"}
+{"current_steps": 10700, "total_steps": 11194, "loss": 0.7211, "lr": 9.620247817535902e-06, "epoch": 0.9605889218062663, "percentage": 95.59, "elapsed_time": "0:31:01", "remaining_time": "0:01:25"}
+{"current_steps": 10710, "total_steps": 11194, "loss": 0.6827, "lr": 9.313726310752581e-06, "epoch": 0.96148666846216, "percentage": 95.68, "elapsed_time": "0:32:30", "remaining_time": "0:01:28"}
+{"current_steps": 10720, "total_steps": 11194, "loss": 0.6761, "lr": 9.00790595084477e-06, "epoch": 0.9623844151180537, "percentage": 95.77, "elapsed_time": "0:33:58", "remaining_time": "0:01:30"}
+{"current_steps": 10730, "total_steps": 11194, "loss": 0.6951, "lr": 8.703077967816944e-06, "epoch": 0.9632821617739474, "percentage": 95.85, "elapsed_time": "0:35:26", "remaining_time": "0:01:31"}
+{"current_steps": 10740, "total_steps": 11194, "loss": 0.7262, "lr": 8.39953264664192e-06, "epoch": 0.9641799084298411, "percentage": 95.94, "elapsed_time": "0:36:55", "remaining_time": "0:01:33"}
+{"current_steps": 10750, "total_steps": 11194, "loss": 0.6866, "lr": 8.097559050825058e-06, "epoch": 0.9650776550857348, "percentage": 96.03, "elapsed_time": "0:38:23", "remaining_time": "0:01:35"}
+{"current_steps": 10760, "total_steps": 11194, "loss": 0.7387, "lr": 7.79744474713166e-06, "epoch": 0.9659754017416285, "percentage": 96.12, "elapsed_time": "0:39:51", "remaining_time": "0:01:36"}
+{"current_steps": 10770, "total_steps": 11194, "loss": 0.779, "lr": 7.499475531739696e-06, "epoch": 0.9668731483975223, "percentage": 96.21, "elapsed_time": "0:41:19", "remaining_time": "0:01:37"}
+{"current_steps": 10780, "total_steps": 11194, "loss": 0.716, "lr": 7.203935158078662e-06, "epoch": 0.967770895053416, "percentage": 96.3, "elapsed_time": "0:42:48", "remaining_time": "0:01:38"}
+{"current_steps": 10790, "total_steps": 11194, "loss": 0.7689, "lr": 6.911105066613709e-06, "epoch": 0.9686686417093097, "percentage": 96.39, "elapsed_time": "0:44:16", "remaining_time": "0:01:39"}
+{"current_steps": 10800, "total_steps": 11194, "loss": 0.7393, "lr": 6.621264116832394e-06, "epoch": 0.9695663883652034, "percentage": 96.48, "elapsed_time": "0:45:44", "remaining_time": "0:01:40"}
+{"current_steps": 10810, "total_steps": 11194, "loss": 0.6807, "lr": 6.334688321689274e-06, "epoch": 0.9704641350210971, "percentage": 96.57, "elapsed_time": "0:47:13", "remaining_time": "0:01:40"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.7452, "lr": 6.051650584761237e-06, "epoch": 0.9713618816769908, "percentage": 96.66, "elapsed_time": "0:49:18", "remaining_time": "0:01:42"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.7143, "lr": 5.772420440363841e-06, "epoch": 0.9722596283328845, "percentage": 96.75, "elapsed_time": "0:50:47", "remaining_time": "0:01:42"}
+{"current_steps": 10840, "total_steps": 11194, "loss": 0.6969, "lr": 5.497263796876207e-06, "epoch": 0.9731573749887782, "percentage": 96.84, "elapsed_time": "0:52:15", "remaining_time": "0:01:42"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.7603, "lr": 6.051650584761237e-06, "epoch": 0.9713618816769908, "percentage": 96.66, "elapsed_time": "0:01:16", "remaining_time": "0:00:02"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.7142, "lr": 5.772420440363841e-06, "epoch": 0.9722596283328845, "percentage": 96.75, "elapsed_time": "0:02:44", "remaining_time": "0:00:05"}
+{"current_steps": 10840, "total_steps": 11194, "loss": 0.6969, "lr": 5.497263796876207e-06, "epoch": 0.9731573749887782, "percentage": 96.84, "elapsed_time": "0:04:13", "remaining_time": "0:00:08"}
+{"current_steps": 10850, "total_steps": 11194, "loss": 0.7809, "lr": 5.226442683518813e-06, "epoch": 0.9740551216446719, "percentage": 96.93, "elapsed_time": "0:05:42", "remaining_time": "0:00:10"}
+{"current_steps": 10860, "total_steps": 11194, "loss": 0.7218, "lr": 4.960215000825432e-06, "epoch": 0.9749528683005656, "percentage": 97.02, "elapsed_time": "0:07:11", "remaining_time": "0:00:13"}
+{"current_steps": 10870, "total_steps": 11194, "loss": 0.7192, "lr": 4.698834275046711e-06, "epoch": 0.9758506149564593, "percentage": 97.11, "elapsed_time": "0:08:40", "remaining_time": "0:00:15"}
+{"current_steps": 10880, "total_steps": 11194, "loss": 0.6673, "lr": 4.442549416719423e-06, "epoch": 0.976748361612353, "percentage": 97.19, "elapsed_time": "0:10:09", "remaining_time": "0:00:17"}
+{"current_steps": 10890, "total_steps": 11194, "loss": 0.8084, "lr": 4.1916044836311565e-06, "epoch": 0.9776461082682467, "percentage": 97.28, "elapsed_time": "0:11:37", "remaining_time": "0:00:19"}
+{"current_steps": 10900, "total_steps": 11194, "loss": 0.6934, "lr": 3.94623844840626e-06, "epoch": 0.9785438549241404, "percentage": 97.37, "elapsed_time": "0:13:06", "remaining_time": "0:00:21"}
+{"current_steps": 10910, "total_steps": 11194, "loss": 0.7194, "lr": 3.7066849709343467e-06, "epoch": 0.9794416015800341, "percentage": 97.46, "elapsed_time": "0:14:35", "remaining_time": "0:00:22"}
+{"current_steps": 10920, "total_steps": 11194, "loss": 0.7763, "lr": 3.473172175858059e-06, "epoch": 0.9803393482359278, "percentage": 97.55, "elapsed_time": "0:16:04", "remaining_time": "0:00:24"}
+{"current_steps": 10930, "total_steps": 11194, "loss": 0.6994, "lr": 3.2459224353320046e-06, "epoch": 0.9812370948918215, "percentage": 97.64, "elapsed_time": "0:17:33", "remaining_time": "0:00:25"}
+{"current_steps": 10940, "total_steps": 11194, "loss": 0.6975, "lr": 3.025152157259735e-06, "epoch": 0.9821348415477152, "percentage": 97.73, "elapsed_time": "0:19:01", "remaining_time": "0:00:26"}
+{"current_steps": 10950, "total_steps": 11194, "loss": 0.707, "lr": 2.811071579210425e-06, "epoch": 0.983032588203609, "percentage": 97.82, "elapsed_time": "0:20:30", "remaining_time": "0:00:27"}
+{"current_steps": 10960, "total_steps": 11194, "loss": 0.673, "lr": 2.603884568211513e-06, "epoch": 0.9839303348595027, "percentage": 97.91, "elapsed_time": "0:21:59", "remaining_time": "0:00:28"}
+{"current_steps": 10970, "total_steps": 11194, "loss": 0.7472, "lr": 2.4037884266079434e-06, "epoch": 0.9848280815153964, "percentage": 98.0, "elapsed_time": "0:23:28", "remaining_time": "0:00:28"}
+{"current_steps": 10980, "total_steps": 11194, "loss": 0.6449, "lr": 2.2109737041728955e-06, "epoch": 0.9857258281712901, "percentage": 98.09, "elapsed_time": "0:25:34", "remaining_time": "0:00:29"}
+{"current_steps": 10990, "total_steps": 11194, "loss": 0.6745, "lr": 2.0256240166489454e-06, "epoch": 0.9866235748271838, "percentage": 98.18, "elapsed_time": "0:27:03", "remaining_time": "0:00:30"}
+{"current_steps": 11000, "total_steps": 11194, "loss": 0.8103, "lr": 1.847915870892413e-06, "epoch": 0.9875213214830775, "percentage": 98.27, "elapsed_time": "0:28:32", "remaining_time": "0:00:30"}
+{"current_steps": 11010, "total_steps": 11194, "loss": 0.6716, "lr": 1.6780184967874715e-06, "epoch": 0.9884190681389712, "percentage": 98.36, "elapsed_time": "0:30:00", "remaining_time": "0:00:30"}
+{"current_steps": 11020, "total_steps": 11194, "loss": 0.8017, "lr": 1.5160936860900294e-06, "epoch": 0.9893168147948649, "percentage": 98.45, "elapsed_time": "0:31:29", "remaining_time": "0:00:29"}
+{"current_steps": 11030, "total_steps": 11194, "loss": 0.8449, "lr": 1.3622956383548762e-06, "epoch": 0.9902145614507586, "percentage": 98.53, "elapsed_time": "0:32:58", "remaining_time": "0:00:29"}
+{"current_steps": 11040, "total_steps": 11194, "loss": 0.6915, "lr": 1.216770814092834e-06, "epoch": 0.9911123081066523, "percentage": 98.62, "elapsed_time": "0:34:27", "remaining_time": "0:00:28"}
+{"current_steps": 11050, "total_steps": 11194, "loss": 0.6839, "lr": 1.0796577952977013e-06, "epoch": 0.992010054762546, "percentage": 98.71, "elapsed_time": "0:35:56", "remaining_time": "0:00:28"}
+{"current_steps": 11060, "total_steps": 11194, "loss": 0.7267, "lr": 9.510871534758705e-07, "epoch": 0.9929078014184397, "percentage": 98.8, "elapsed_time": "0:37:25", "remaining_time": "0:00:27"}
+{"current_steps": 11070, "total_steps": 11194, "loss": 0.729, "lr": 8.311813253042334e-07, "epoch": 0.9938055480743334, "percentage": 98.89, "elapsed_time": "0:38:53", "remaining_time": "0:00:26"}
+{"current_steps": 11080, "total_steps": 11194, "loss": 0.7448, "lr": 7.200544960348308e-07, "epoch": 0.9947032947302271, "percentage": 98.98, "elapsed_time": "0:40:22", "remaining_time": "0:00:24"}
+{"current_steps": 11090, "total_steps": 11194, "loss": 0.7767, "lr": 6.178124907572545e-07, "epoch": 0.9956010413861208, "percentage": 99.07, "elapsed_time": "0:41:51", "remaining_time": "0:00:23"}
+{"current_steps": 11100, "total_steps": 11194, "loss": 0.7806, "lr": 5.245526736223528e-07, "epoch": 0.9964987880420145, "percentage": 99.16, "elapsed_time": "0:43:20", "remaining_time": "0:00:22"}
+{"current_steps": 11110, "total_steps": 11194, "loss": 0.6977, "lr": 4.403638551232205e-07, "epoch": 0.9973965346979082, "percentage": 99.25, "elapsed_time": "0:44:49", "remaining_time": "0:00:20"}
+{"current_steps": 11120, "total_steps": 11194, "loss": 0.6909, "lr": 3.6532620752175616e-07, "epoch": 0.9982942813538019, "percentage": 99.34, "elapsed_time": "0:46:17", "remaining_time": "0:00:18"}
+{"current_steps": 11130, "total_steps": 11194, "loss": 0.7274, "lr": 2.9951118850133247e-07, "epoch": 0.9991920280096956, "percentage": 99.43, "elapsed_time": "0:47:46", "remaining_time": "0:00:16"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.7357, "lr": 2.429814731182774e-07, "epoch": 1.0000897746655895, "percentage": 99.52, "elapsed_time": "0:49:54", "remaining_time": "0:00:14"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.7357, "lr": 2.429814731182774e-07, "epoch": 1.0000897746655895, "percentage": 99.52, "elapsed_time": "0:01:33", "remaining_time": "0:00:00"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.5747, "lr": 1.957908941169813e-07, "epoch": 1.0009875213214832, "percentage": 99.61, "elapsed_time": "0:03:02", "remaining_time": "0:00:00"}
+{"current_steps": 11160, "total_steps": 11194, "loss": 0.5986, "lr": 1.5798439066545254e-07, "epoch": 1.0018852679773769, "percentage": 99.7, "elapsed_time": "0:04:31", "remaining_time": "0:00:00"}
+{"current_steps": 11170, "total_steps": 11194, "loss": 0.6214, "lr": 1.2959796556014471e-07, "epoch": 1.0027830146332706, "percentage": 99.79, "elapsed_time": "0:06:00", "remaining_time": "0:00:00"}
+{"current_steps": 11180, "total_steps": 11194, "loss": 0.6295, "lr": 1.1065865094081884e-07, "epoch": 1.0036807612891643, "percentage": 99.87, "elapsed_time": "0:07:29", "remaining_time": "0:00:00"}
+{"current_steps": 11190, "total_steps": 11194, "loss": 0.6283, "lr": 1.011844825480734e-07, "epoch": 1.004578507945058, "percentage": 99.96, "elapsed_time": "0:08:57", "remaining_time": "0:00:00"}
+{"current_steps": 11194, "total_steps": 11194, "epoch": 1.0049376066074154, "percentage": 100.0, "elapsed_time": "0:10:10", "remaining_time": "0:00:00"}

g1_0_baseline_original_code/500M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.5024688033037077,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.028161267840651417,
+    "train_runtime": 1789.279,
+    "train_samples_per_second": 37.537,
+    "train_steps_per_second": 3.128
+}

g1_0_baseline_original_code/500M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g1_0_baseline_original_code/500M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g1_0_baseline_original_code/500M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g1_0_baseline_original_code/500M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g1_0_baseline_original_code/500M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g1_0_baseline_original_code/500M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g1_0_baseline_original_code/500M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.5024688033037077,
+    "total_flos": 2.2737574718397743e+19,
+    "train_loss": 0.028161267840651417,
+    "train_runtime": 1789.279,
+    "train_samples_per_second": 37.537,
+    "train_steps_per_second": 3.128
+}

g1_0_baseline_original_code/500M/trainer_log.jsonl

@@ -0,0 +1,55 @@
+{"current_steps": 5090, "total_steps": 5597, "loss": 0.7741, "lr": 1.9999810479633703e-05, "epoch": 0.45695304784989677, "percentage": 90.94, "elapsed_time": "0:00:32", "remaining_time": "0:00:03"}
+{"current_steps": 5100, "total_steps": 5597, "loss": 0.8417, "lr": 1.997707677020786e-05, "epoch": 0.45785079450579047, "percentage": 91.12, "elapsed_time": "0:02:02", "remaining_time": "0:00:11"}
+{"current_steps": 5110, "total_steps": 5597, "loss": 0.8694, "lr": 1.9916538196124637e-05, "epoch": 0.4587485411616842, "percentage": 91.3, "elapsed_time": "0:03:31", "remaining_time": "0:00:20"}
+{"current_steps": 5120, "total_steps": 5597, "loss": 0.8408, "lr": 1.9818425303863402e-05, "epoch": 0.4596462878175779, "percentage": 91.48, "elapsed_time": "0:05:01", "remaining_time": "0:00:28"}
+{"current_steps": 5130, "total_steps": 5597, "loss": 0.8755, "lr": 1.9683111732582887e-05, "epoch": 0.4605440344734716, "percentage": 91.66, "elapsed_time": "0:06:30", "remaining_time": "0:00:35"}
+{"current_steps": 5140, "total_steps": 5597, "loss": 0.7917, "lr": 1.951111279120706e-05, "epoch": 0.4614417811293653, "percentage": 91.83, "elapsed_time": "0:07:59", "remaining_time": "0:00:42"}
+{"current_steps": 5150, "total_steps": 5597, "loss": 0.7847, "lr": 1.9303083495996028e-05, "epoch": 0.462339527785259, "percentage": 92.01, "elapsed_time": "0:09:29", "remaining_time": "0:00:49"}
+{"current_steps": 5160, "total_steps": 5597, "loss": 0.8769, "lr": 1.9059816076075482e-05, "epoch": 0.4632372744411527, "percentage": 92.19, "elapsed_time": "0:10:58", "remaining_time": "0:00:55"}
+{"current_steps": 5170, "total_steps": 5597, "loss": 0.7992, "lr": 1.8782236956424142e-05, "epoch": 0.4641350210970464, "percentage": 92.37, "elapsed_time": "0:12:28", "remaining_time": "0:01:01"}
+{"current_steps": 5180, "total_steps": 5597, "loss": 0.8254, "lr": 1.8471403229808897e-05, "epoch": 0.4650327677529401, "percentage": 92.55, "elapsed_time": "0:13:57", "remaining_time": "0:01:07"}
+{"current_steps": 5190, "total_steps": 5597, "loss": 0.8724, "lr": 1.8128498631103333e-05, "epoch": 0.4659305144088338, "percentage": 92.73, "elapsed_time": "0:15:26", "remaining_time": "0:01:12"}
+{"current_steps": 5200, "total_steps": 5597, "loss": 0.8527, "lr": 1.7754829029320507e-05, "epoch": 0.4668282610647275, "percentage": 92.91, "elapsed_time": "0:16:55", "remaining_time": "0:01:17"}
+{"current_steps": 5210, "total_steps": 5597, "loss": 0.7918, "lr": 1.7351817454527473e-05, "epoch": 0.46772600772062123, "percentage": 93.09, "elapsed_time": "0:18:25", "remaining_time": "0:01:22"}
+{"current_steps": 5220, "total_steps": 5597, "loss": 0.8084, "lr": 1.6920998678580345e-05, "epoch": 0.46862375437651493, "percentage": 93.26, "elapsed_time": "0:19:54", "remaining_time": "0:01:26"}
+{"current_steps": 5230, "total_steps": 5597, "loss": 0.836, "lr": 1.6464013370317854e-05, "epoch": 0.46952150103240864, "percentage": 93.44, "elapsed_time": "0:21:23", "remaining_time": "0:01:30"}
+{"current_steps": 5240, "total_steps": 5597, "loss": 0.841, "lr": 1.5982601847471915e-05, "epoch": 0.47041924768830234, "percentage": 93.62, "elapsed_time": "0:22:52", "remaining_time": "0:01:33"}
+{"current_steps": 5250, "total_steps": 5597, "loss": 0.8971, "lr": 1.5478597449089553e-05, "epoch": 0.47131699434419605, "percentage": 93.8, "elapsed_time": "0:25:00", "remaining_time": "0:01:39"}
+{"current_steps": 5260, "total_steps": 5597, "loss": 0.8462, "lr": 1.4953919553705725e-05, "epoch": 0.47221474100008975, "percentage": 93.98, "elapsed_time": "0:26:30", "remaining_time": "0:01:41"}
+{"current_steps": 5270, "total_steps": 5597, "loss": 0.8469, "lr": 1.441056626985559e-05, "epoch": 0.47311248765598346, "percentage": 94.16, "elapsed_time": "0:27:59", "remaining_time": "0:01:44"}
+{"current_steps": 5280, "total_steps": 5597, "loss": 0.799, "lr": 1.3850606826762668e-05, "epoch": 0.47401023431187717, "percentage": 94.34, "elapsed_time": "0:29:29", "remaining_time": "0:01:46"}
+{"current_steps": 5290, "total_steps": 5597, "loss": 0.8349, "lr": 1.3276173694181128e-05, "epoch": 0.47490798096777087, "percentage": 94.51, "elapsed_time": "0:30:58", "remaining_time": "0:01:47"}
+{"current_steps": 5300, "total_steps": 5597, "loss": 0.7874, "lr": 1.2689454461401822e-05, "epoch": 0.4758057276236646, "percentage": 94.69, "elapsed_time": "0:32:28", "remaining_time": "0:01:49"}
+{"current_steps": 5310, "total_steps": 5597, "loss": 0.7854, "lr": 1.2092683506348939e-05, "epoch": 0.4767034742795583, "percentage": 94.87, "elapsed_time": "0:33:57", "remaining_time": "0:01:50"}
+{"current_steps": 5320, "total_steps": 5597, "loss": 0.8265, "lr": 1.14881334864935e-05, "epoch": 0.47760122093545204, "percentage": 95.05, "elapsed_time": "0:35:26", "remaining_time": "0:01:50"}
+{"current_steps": 5330, "total_steps": 5597, "loss": 0.8637, "lr": 1.0878106683988481e-05, "epoch": 0.47849896759134575, "percentage": 95.23, "elapsed_time": "0:36:56", "remaining_time": "0:01:51"}
+{"current_steps": 5340, "total_steps": 5597, "loss": 0.7507, "lr": 1.026492623798551e-05, "epoch": 0.47939671424723945, "percentage": 95.41, "elapsed_time": "0:38:25", "remaining_time": "0:01:50"}
+{"current_steps": 5350, "total_steps": 5597, "loss": 0.8437, "lr": 9.650927297522743e-06, "epoch": 0.48029446090313316, "percentage": 95.59, "elapsed_time": "0:39:55", "remaining_time": "0:01:50"}
+{"current_steps": 5360, "total_steps": 5597, "loss": 0.8547, "lr": 9.038448128676012e-06, "epoch": 0.48119220755902686, "percentage": 95.77, "elapsed_time": "0:41:24", "remaining_time": "0:01:49"}
+{"current_steps": 5370, "total_steps": 5597, "loss": 0.9039, "lr": 8.42982120983949e-06, "epoch": 0.48208995421492057, "percentage": 95.94, "elapsed_time": "0:42:53", "remaining_time": "0:01:48"}
+{"current_steps": 5380, "total_steps": 5597, "loss": 0.7911, "lr": 7.827364349047303e-06, "epoch": 0.4829877008708143, "percentage": 96.12, "elapsed_time": "0:44:23", "remaining_time": "0:01:47"}
+{"current_steps": 5390, "total_steps": 5597, "loss": 0.804, "lr": 7.233371857163706e-06, "epoch": 0.483885447526708, "percentage": 96.3, "elapsed_time": "0:45:52", "remaining_time": "0:01:45"}
+{"current_steps": 5400, "total_steps": 5597, "loss": 0.7966, "lr": 6.650105810556492e-06, "epoch": 0.4847831941826017, "percentage": 96.48, "elapsed_time": "0:47:22", "remaining_time": "0:01:43"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.7825, "lr": 6.079787436527681e-06, "epoch": 0.4856809408384954, "percentage": 96.66, "elapsed_time": "0:49:29", "remaining_time": "0:01:42"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.8753, "lr": 5.524588654307961e-06, "epoch": 0.4865786874943891, "percentage": 96.84, "elapsed_time": "0:50:58", "remaining_time": "0:01:39"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.8486, "lr": 4.986623803828985e-06, "epoch": 0.4874764341502828, "percentage": 97.02, "elapsed_time": "0:52:27", "remaining_time": "0:01:36"}
+{"current_steps": 5410, "total_steps": 5597, "loss": 0.7768, "lr": 6.079787436527681e-06, "epoch": 0.4856809408384954, "percentage": 96.66, "elapsed_time": "0:00:40", "remaining_time": "0:00:01"}
+{"current_steps": 5420, "total_steps": 5597, "loss": 0.8754, "lr": 5.524588654307961e-06, "epoch": 0.4865786874943891, "percentage": 96.84, "elapsed_time": "0:02:09", "remaining_time": "0:00:04"}
+{"current_steps": 5430, "total_steps": 5597, "loss": 0.8486, "lr": 4.986623803828985e-06, "epoch": 0.4874764341502828, "percentage": 97.02, "elapsed_time": "0:03:39", "remaining_time": "0:00:06"}
+{"current_steps": 5440, "total_steps": 5597, "loss": 0.8041, "lr": 4.467941593772423e-06, "epoch": 0.4883741808061765, "percentage": 97.19, "elapsed_time": "0:05:08", "remaining_time": "0:00:08"}
+{"current_steps": 5450, "total_steps": 5597, "loss": 0.8244, "lr": 3.9705172995597606e-06, "epoch": 0.4892719274620702, "percentage": 97.37, "elapsed_time": "0:06:38", "remaining_time": "0:00:10"}
+{"current_steps": 5460, "total_steps": 5597, "loss": 0.8175, "lr": 3.4962452409948346e-06, "epoch": 0.4901696741179639, "percentage": 97.55, "elapsed_time": "0:08:07", "remaining_time": "0:00:12"}
+{"current_steps": 5470, "total_steps": 5597, "loss": 0.7875, "lr": 3.046931568206211e-06, "epoch": 0.4910674207738576, "percentage": 97.73, "elapsed_time": "0:09:37", "remaining_time": "0:00:13"}
+{"current_steps": 5480, "total_steps": 5597, "loss": 0.8289, "lr": 2.624287383362404e-06, "epoch": 0.4919651674297513, "percentage": 97.91, "elapsed_time": "0:11:06", "remaining_time": "0:00:14"}
+{"current_steps": 5490, "total_steps": 5597, "loss": 0.8606, "lr": 2.2299222243541845e-06, "epoch": 0.49286291408564503, "percentage": 98.09, "elapsed_time": "0:12:36", "remaining_time": "0:00:14"}
+{"current_steps": 5500, "total_steps": 5597, "loss": 0.7984, "lr": 1.8653379352598246e-06, "epoch": 0.49376066074153874, "percentage": 98.27, "elapsed_time": "0:14:05", "remaining_time": "0:00:14"}
+{"current_steps": 5510, "total_steps": 5597, "loss": 0.881, "lr": 1.5319229469361075e-06, "epoch": 0.49465840739743244, "percentage": 98.45, "elapsed_time": "0:15:35", "remaining_time": "0:00:14"}
+{"current_steps": 5520, "total_steps": 5597, "loss": 0.8711, "lr": 1.2309469895160917e-06, "epoch": 0.49555615405332615, "percentage": 98.62, "elapsed_time": "0:17:04", "remaining_time": "0:00:14"}
+{"current_steps": 5530, "total_steps": 5597, "loss": 0.8399, "lr": 9.635562569497974e-07, "epoch": 0.49645390070921985, "percentage": 98.8, "elapsed_time": "0:18:34", "remaining_time": "0:00:13"}
+{"current_steps": 5540, "total_steps": 5597, "loss": 0.8138, "lr": 7.307690420024867e-07, "epoch": 0.49735164736511356, "percentage": 98.98, "elapsed_time": "0:20:03", "remaining_time": "0:00:12"}
+{"current_steps": 5550, "total_steps": 5597, "loss": 0.8633, "lr": 5.334718583335942e-07, "epoch": 0.49824939402100726, "percentage": 99.16, "elapsed_time": "0:21:33", "remaining_time": "0:00:10"}
+{"current_steps": 5560, "total_steps": 5597, "loss": 0.8252, "lr": 3.7241606442442763e-07, "epoch": 0.49914714067690097, "percentage": 99.34, "elapsed_time": "0:23:02", "remaining_time": "0:00:09"}
+{"current_steps": 5570, "total_steps": 5597, "loss": 0.8292, "lr": 2.482150022115879e-07, "epoch": 0.5000448873327947, "percentage": 99.52, "elapsed_time": "0:25:10", "remaining_time": "0:00:07"}
+{"current_steps": 5580, "total_steps": 5597, "loss": 0.819, "lr": 1.6134166132293174e-07, "epoch": 0.5009426339886884, "percentage": 99.7, "elapsed_time": "0:26:39", "remaining_time": "0:00:04"}
+{"current_steps": 5590, "total_steps": 5597, "loss": 0.7834, "lr": 1.1212687781127357e-07, "epoch": 0.5018403806445821, "percentage": 99.87, "elapsed_time": "0:28:09", "remaining_time": "0:00:02"}
+{"current_steps": 5597, "total_steps": 5597, "epoch": 0.5024688033037077, "percentage": 100.0, "elapsed_time": "0:29:49", "remaining_time": "0:00:00"}

g1_0_baseline_original_code/500M/trainer_state.json

@@ -0,0 +1,3956 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5024688033037077,
+  "eval_steps": 500,
+  "global_step": 5597,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008977466558937068,
+      "grad_norm": 7.634389400482178,
+      "learning_rate": 1.1061797752808989e-06,
+      "loss": 1.9314,
+      "step": 10
+    },
+    {
+      "epoch": 0.0017954933117874136,
+      "grad_norm": 3.166454553604126,
+      "learning_rate": 2.2241573033707866e-06,
+      "loss": 1.8945,
+      "step": 20
+    },
+    {
+      "epoch": 0.00269323996768112,
+      "grad_norm": 1.6780861616134644,
+      "learning_rate": 3.342134831460675e-06,
+      "loss": 1.6996,
+      "step": 30
+    },
+    {
+      "epoch": 0.003590986623574827,
+      "grad_norm": 1.238744854927063,
+      "learning_rate": 4.4601123595505624e-06,
+      "loss": 1.5209,
+      "step": 40
+    },
+    {
+      "epoch": 0.004488733279468534,
+      "grad_norm": 0.8214995861053467,
+      "learning_rate": 5.5780898876404495e-06,
+      "loss": 1.3707,
+      "step": 50
+    },
+    {
+      "epoch": 0.00538647993536224,
+      "grad_norm": 0.6225201487541199,
+      "learning_rate": 6.6960674157303374e-06,
+      "loss": 1.2258,
+      "step": 60
+    },
+    {
+      "epoch": 0.006284226591255947,
+      "grad_norm": 0.6041092276573181,
+      "learning_rate": 7.814044943820226e-06,
+      "loss": 1.1689,
+      "step": 70
+    },
+    {
+      "epoch": 0.007181973247149654,
+      "grad_norm": 0.5930420756340027,
+      "learning_rate": 8.932022471910112e-06,
+      "loss": 1.1735,
+      "step": 80
+    },
+    {
+      "epoch": 0.008079719903043361,
+      "grad_norm": 0.603300929069519,
+      "learning_rate": 1.005e-05,
+      "loss": 1.1347,
+      "step": 90
+    },
+    {
+      "epoch": 0.008977466558937068,
+      "grad_norm": 0.5618454217910767,
+      "learning_rate": 1.1167977528089888e-05,
+      "loss": 1.1165,
+      "step": 100
+    },
+    {
+      "epoch": 0.009875213214830775,
+      "grad_norm": 0.5842673182487488,
+      "learning_rate": 1.2285955056179776e-05,
+      "loss": 1.0947,
+      "step": 110
+    },
+    {
+      "epoch": 0.01077295987072448,
+      "grad_norm": 0.5741180777549744,
+      "learning_rate": 1.3403932584269664e-05,
+      "loss": 1.1028,
+      "step": 120
+    },
+    {
+      "epoch": 0.011670706526618188,
+      "grad_norm": 0.5997037291526794,
+      "learning_rate": 1.4521910112359552e-05,
+      "loss": 1.1276,
+      "step": 130
+    },
+    {
+      "epoch": 0.012568453182511895,
+      "grad_norm": 0.5892444849014282,
+      "learning_rate": 1.563988764044944e-05,
+      "loss": 1.1735,
+      "step": 140
+    },
+    {
+      "epoch": 0.013466199838405602,
+      "grad_norm": 0.5162788033485413,
+      "learning_rate": 1.6757865168539328e-05,
+      "loss": 1.0822,
+      "step": 150
+    },
+    {
+      "epoch": 0.014363946494299309,
+      "grad_norm": 0.6133973598480225,
+      "learning_rate": 1.7875842696629214e-05,
+      "loss": 1.2064,
+      "step": 160
+    },
+    {
+      "epoch": 0.015261693150193016,
+      "grad_norm": 0.5716113448143005,
+      "learning_rate": 1.89938202247191e-05,
+      "loss": 1.1207,
+      "step": 170
+    },
+    {
+      "epoch": 0.016159439806086723,
+      "grad_norm": 0.8888991475105286,
+      "learning_rate": 2e-05,
+      "loss": 1.1478,
+      "step": 180
+    },
+    {
+      "epoch": 0.017057186461980428,
+      "grad_norm": 0.558070182800293,
+      "learning_rate": 2e-05,
+      "loss": 1.099,
+      "step": 190
+    },
+    {
+      "epoch": 0.017954933117874137,
+      "grad_norm": 0.6470697522163391,
+      "learning_rate": 2e-05,
+      "loss": 1.0708,
+      "step": 200
+    },
+    {
+      "epoch": 0.018852679773767842,
+      "grad_norm": 0.5357931852340698,
+      "learning_rate": 2e-05,
+      "loss": 1.1295,
+      "step": 210
+    },
+    {
+      "epoch": 0.01975042642966155,
+      "grad_norm": 0.5657357573509216,
+      "learning_rate": 2e-05,
+      "loss": 1.1604,
+      "step": 220
+    },
+    {
+      "epoch": 0.020648173085555256,
+      "grad_norm": 0.6352828741073608,
+      "learning_rate": 2e-05,
+      "loss": 1.119,
+      "step": 230
+    },
+    {
+      "epoch": 0.02154591974144896,
+      "grad_norm": 0.5701799988746643,
+      "learning_rate": 2e-05,
+      "loss": 1.0867,
+      "step": 240
+    },
+    {
+      "epoch": 0.02244366639734267,
+      "grad_norm": 0.54180508852005,
+      "learning_rate": 2e-05,
+      "loss": 1.0811,
+      "step": 250
+    },
+    {
+      "epoch": 0.023341413053236375,
+      "grad_norm": 0.5608647465705872,
+      "learning_rate": 2e-05,
+      "loss": 1.0937,
+      "step": 260
+    },
+    {
+      "epoch": 0.024239159709130084,
+      "grad_norm": 0.5483505129814148,
+      "learning_rate": 2e-05,
+      "loss": 1.0812,
+      "step": 270
+    },
+    {
+      "epoch": 0.02513690636502379,
+      "grad_norm": 0.5331147909164429,
+      "learning_rate": 2e-05,
+      "loss": 1.0576,
+      "step": 280
+    },
+    {
+      "epoch": 0.026034653020917498,
+      "grad_norm": 0.8958403468132019,
+      "learning_rate": 2e-05,
+      "loss": 1.1206,
+      "step": 290
+    },
+    {
+      "epoch": 0.026932399676811204,
+      "grad_norm": 0.5771209001541138,
+      "learning_rate": 2e-05,
+      "loss": 1.14,
+      "step": 300
+    },
+    {
+      "epoch": 0.027830146332704912,
+      "grad_norm": 0.508565366268158,
+      "learning_rate": 2e-05,
+      "loss": 1.0731,
+      "step": 310
+    },
+    {
+      "epoch": 0.028727892988598618,
+      "grad_norm": 0.6553564667701721,
+      "learning_rate": 2e-05,
+      "loss": 1.1244,
+      "step": 320
+    },
+    {
+      "epoch": 0.029625639644492323,
+      "grad_norm": 0.5307732820510864,
+      "learning_rate": 2e-05,
+      "loss": 1.109,
+      "step": 330
+    },
+    {
+      "epoch": 0.03052338630038603,
+      "grad_norm": 0.5320704579353333,
+      "learning_rate": 2e-05,
+      "loss": 1.0395,
+      "step": 340
+    },
+    {
+      "epoch": 0.03142113295627974,
+      "grad_norm": 0.48801037669181824,
+      "learning_rate": 2e-05,
+      "loss": 1.0765,
+      "step": 350
+    },
+    {
+      "epoch": 0.032318879612173446,
+      "grad_norm": 0.5760318636894226,
+      "learning_rate": 2e-05,
+      "loss": 1.0601,
+      "step": 360
+    },
+    {
+      "epoch": 0.03321662626806715,
+      "grad_norm": 0.5792495608329773,
+      "learning_rate": 2e-05,
+      "loss": 1.1067,
+      "step": 370
+    },
+    {
+      "epoch": 0.034114372923960856,
+      "grad_norm": 0.604045033454895,
+      "learning_rate": 2e-05,
+      "loss": 1.0792,
+      "step": 380
+    },
+    {
+      "epoch": 0.03501211957985457,
+      "grad_norm": 0.5748013854026794,
+      "learning_rate": 2e-05,
+      "loss": 1.1062,
+      "step": 390
+    },
+    {
+      "epoch": 0.035909866235748274,
+      "grad_norm": 0.5119761228561401,
+      "learning_rate": 2e-05,
+      "loss": 1.0289,
+      "step": 400
+    },
+    {
+      "epoch": 0.03680761289164198,
+      "grad_norm": 0.5493387579917908,
+      "learning_rate": 2e-05,
+      "loss": 1.0338,
+      "step": 410
+    },
+    {
+      "epoch": 0.037705359547535684,
+      "grad_norm": 0.6864805221557617,
+      "learning_rate": 2e-05,
+      "loss": 1.1149,
+      "step": 420
+    },
+    {
+      "epoch": 0.03860310620342939,
+      "grad_norm": 0.5241488218307495,
+      "learning_rate": 2e-05,
+      "loss": 1.0708,
+      "step": 430
+    },
+    {
+      "epoch": 0.0395008528593231,
+      "grad_norm": 0.5357916355133057,
+      "learning_rate": 2e-05,
+      "loss": 1.0752,
+      "step": 440
+    },
+    {
+      "epoch": 0.04039859951521681,
+      "grad_norm": 0.9240272641181946,
+      "learning_rate": 2e-05,
+      "loss": 1.0632,
+      "step": 450
+    },
+    {
+      "epoch": 0.04129634617111051,
+      "grad_norm": 0.5168442726135254,
+      "learning_rate": 2e-05,
+      "loss": 1.0794,
+      "step": 460
+    },
+    {
+      "epoch": 0.04219409282700422,
+      "grad_norm": 0.5578870177268982,
+      "learning_rate": 2e-05,
+      "loss": 1.0579,
+      "step": 470
+    },
+    {
+      "epoch": 0.04309183948289792,
+      "grad_norm": 4.224264144897461,
+      "learning_rate": 2e-05,
+      "loss": 1.1145,
+      "step": 480
+    },
+    {
+      "epoch": 0.043989586138791635,
+      "grad_norm": 0.5236369371414185,
+      "learning_rate": 2e-05,
+      "loss": 1.0754,
+      "step": 490
+    },
+    {
+      "epoch": 0.04488733279468534,
+      "grad_norm": 0.5369593501091003,
+      "learning_rate": 2e-05,
+      "loss": 1.0746,
+      "step": 500
+    },
+    {
+      "epoch": 0.045785079450579046,
+      "grad_norm": 0.5629836916923523,
+      "learning_rate": 2e-05,
+      "loss": 1.0203,
+      "step": 510
+    },
+    {
+      "epoch": 0.04668282610647275,
+      "grad_norm": 1.1909027099609375,
+      "learning_rate": 2e-05,
+      "loss": 1.0484,
+      "step": 520
+    },
+    {
+      "epoch": 0.04758057276236646,
+      "grad_norm": 0.5194602608680725,
+      "learning_rate": 2e-05,
+      "loss": 1.1053,
+      "step": 530
+    },
+    {
+      "epoch": 0.04847831941826017,
+      "grad_norm": 0.49282965064048767,
+      "learning_rate": 2e-05,
+      "loss": 1.0526,
+      "step": 540
+    },
+    {
+      "epoch": 0.049376066074153874,
+      "grad_norm": 0.5095282196998596,
+      "learning_rate": 2e-05,
+      "loss": 1.0829,
+      "step": 550
+    },
+    {
+      "epoch": 0.05027381273004758,
+      "grad_norm": 0.574981153011322,
+      "learning_rate": 2e-05,
+      "loss": 1.0399,
+      "step": 560
+    },
+    {
+      "epoch": 0.051171559385941284,
+      "grad_norm": 0.5354433655738831,
+      "learning_rate": 2e-05,
+      "loss": 1.0195,
+      "step": 570
+    },
+    {
+      "epoch": 0.052069306041834997,
+      "grad_norm": 0.5005495548248291,
+      "learning_rate": 2e-05,
+      "loss": 1.0515,
+      "step": 580
+    },
+    {
+      "epoch": 0.0529670526977287,
+      "grad_norm": 0.4863434135913849,
+      "learning_rate": 2e-05,
+      "loss": 1.0854,
+      "step": 590
+    },
+    {
+      "epoch": 0.05386479935362241,
+      "grad_norm": 0.8269434571266174,
+      "learning_rate": 2e-05,
+      "loss": 1.0621,
+      "step": 600
+    },
+    {
+      "epoch": 0.05476254600951611,
+      "grad_norm": 0.5037260055541992,
+      "learning_rate": 2e-05,
+      "loss": 1.0329,
+      "step": 610
+    },
+    {
+      "epoch": 0.055660292665409825,
+      "grad_norm": 0.49951377511024475,
+      "learning_rate": 2e-05,
+      "loss": 1.0823,
+      "step": 620
+    },
+    {
+      "epoch": 0.05655803932130353,
+      "grad_norm": 0.5041629076004028,
+      "learning_rate": 2e-05,
+      "loss": 1.0681,
+      "step": 630
+    },
+    {
+      "epoch": 0.057455785977197235,
+      "grad_norm": 0.4688110947608948,
+      "learning_rate": 2e-05,
+      "loss": 1.0772,
+      "step": 640
+    },
+    {
+      "epoch": 0.05835353263309094,
+      "grad_norm": 0.5006560683250427,
+      "learning_rate": 2e-05,
+      "loss": 1.0695,
+      "step": 650
+    },
+    {
+      "epoch": 0.059251279288984646,
+      "grad_norm": 0.5676978230476379,
+      "learning_rate": 2e-05,
+      "loss": 1.0395,
+      "step": 660
+    },
+    {
+      "epoch": 0.06014902594487836,
+      "grad_norm": 1.2871171236038208,
+      "learning_rate": 2e-05,
+      "loss": 1.0843,
+      "step": 670
+    },
+    {
+      "epoch": 0.06104677260077206,
+      "grad_norm": 0.701192319393158,
+      "learning_rate": 2e-05,
+      "loss": 1.0473,
+      "step": 680
+    },
+    {
+      "epoch": 0.06194451925666577,
+      "grad_norm": 0.7122907042503357,
+      "learning_rate": 2e-05,
+      "loss": 1.0651,
+      "step": 690
+    },
+    {
+      "epoch": 0.06284226591255948,
+      "grad_norm": 0.47654181718826294,
+      "learning_rate": 2e-05,
+      "loss": 1.0682,
+      "step": 700
+    },
+    {
+      "epoch": 0.06374001256845319,
+      "grad_norm": 0.513289213180542,
+      "learning_rate": 2e-05,
+      "loss": 1.0388,
+      "step": 710
+    },
+    {
+      "epoch": 0.06463775922434689,
+      "grad_norm": 0.6258441209793091,
+      "learning_rate": 2e-05,
+      "loss": 1.0492,
+      "step": 720
+    },
+    {
+      "epoch": 0.0655355058802406,
+      "grad_norm": 0.5013813376426697,
+      "learning_rate": 2e-05,
+      "loss": 1.0458,
+      "step": 730
+    },
+    {
+      "epoch": 0.0664332525361343,
+      "grad_norm": 0.5458043217658997,
+      "learning_rate": 2e-05,
+      "loss": 1.0525,
+      "step": 740
+    },
+    {
+      "epoch": 0.06733099919202801,
+      "grad_norm": 0.5081648230552673,
+      "learning_rate": 2e-05,
+      "loss": 1.0201,
+      "step": 750
+    },
+    {
+      "epoch": 0.06822874584792171,
+      "grad_norm": 0.49785295128822327,
+      "learning_rate": 2e-05,
+      "loss": 1.1267,
+      "step": 760
+    },
+    {
+      "epoch": 0.06912649250381542,
+      "grad_norm": 0.5245663523674011,
+      "learning_rate": 2e-05,
+      "loss": 1.0173,
+      "step": 770
+    },
+    {
+      "epoch": 0.07002423915970914,
+      "grad_norm": 0.48743847012519836,
+      "learning_rate": 2e-05,
+      "loss": 1.0074,
+      "step": 780
+    },
+    {
+      "epoch": 0.07092198581560284,
+      "grad_norm": 0.6617560982704163,
+      "learning_rate": 2e-05,
+      "loss": 1.044,
+      "step": 790
+    },
+    {
+      "epoch": 0.07181973247149655,
+      "grad_norm": 0.5401202440261841,
+      "learning_rate": 2e-05,
+      "loss": 1.0229,
+      "step": 800
+    },
+    {
+      "epoch": 0.07271747912739025,
+      "grad_norm": 0.543354868888855,
+      "learning_rate": 2e-05,
+      "loss": 1.0567,
+      "step": 810
+    },
+    {
+      "epoch": 0.07361522578328396,
+      "grad_norm": 1.1279566287994385,
+      "learning_rate": 2e-05,
+      "loss": 1.0055,
+      "step": 820
+    },
+    {
+      "epoch": 0.07451297243917766,
+      "grad_norm": 0.5519621968269348,
+      "learning_rate": 2e-05,
+      "loss": 1.0213,
+      "step": 830
+    },
+    {
+      "epoch": 0.07541071909507137,
+      "grad_norm": 1.1007333993911743,
+      "learning_rate": 2e-05,
+      "loss": 1.0484,
+      "step": 840
+    },
+    {
+      "epoch": 0.07630846575096507,
+      "grad_norm": 0.47256144881248474,
+      "learning_rate": 2e-05,
+      "loss": 1.0677,
+      "step": 850
+    },
+    {
+      "epoch": 0.07720621240685878,
+      "grad_norm": 0.4924080967903137,
+      "learning_rate": 2e-05,
+      "loss": 1.0573,
+      "step": 860
+    },
+    {
+      "epoch": 0.07810395906275248,
+      "grad_norm": 0.5096149444580078,
+      "learning_rate": 2e-05,
+      "loss": 1.1057,
+      "step": 870
+    },
+    {
+      "epoch": 0.0790017057186462,
+      "grad_norm": 0.5318610072135925,
+      "learning_rate": 2e-05,
+      "loss": 1.0146,
+      "step": 880
+    },
+    {
+      "epoch": 0.07989945237453991,
+      "grad_norm": 0.675080418586731,
+      "learning_rate": 2e-05,
+      "loss": 1.0599,
+      "step": 890
+    },
+    {
+      "epoch": 0.08079719903043361,
+      "grad_norm": 0.47846636176109314,
+      "learning_rate": 2e-05,
+      "loss": 1.0417,
+      "step": 900
+    },
+    {
+      "epoch": 0.08169494568632732,
+      "grad_norm": 0.4796086847782135,
+      "learning_rate": 2e-05,
+      "loss": 1.0107,
+      "step": 910
+    },
+    {
+      "epoch": 0.08259269234222102,
+      "grad_norm": 0.47223231196403503,
+      "learning_rate": 2e-05,
+      "loss": 1.0659,
+      "step": 920
+    },
+    {
+      "epoch": 0.08349043899811473,
+      "grad_norm": 0.5378100275993347,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 930
+    },
+    {
+      "epoch": 0.08438818565400844,
+      "grad_norm": 0.5030729174613953,
+      "learning_rate": 2e-05,
+      "loss": 1.0183,
+      "step": 940
+    },
+    {
+      "epoch": 0.08528593230990214,
+      "grad_norm": 0.5262132287025452,
+      "learning_rate": 2e-05,
+      "loss": 1.0381,
+      "step": 950
+    },
+    {
+      "epoch": 0.08618367896579585,
+      "grad_norm": 0.5390843749046326,
+      "learning_rate": 2e-05,
+      "loss": 1.01,
+      "step": 960
+    },
+    {
+      "epoch": 0.08708142562168956,
+      "grad_norm": 0.5111972689628601,
+      "learning_rate": 2e-05,
+      "loss": 1.0757,
+      "step": 970
+    },
+    {
+      "epoch": 0.08797917227758327,
+      "grad_norm": 0.5065791010856628,
+      "learning_rate": 2e-05,
+      "loss": 1.0301,
+      "step": 980
+    },
+    {
+      "epoch": 0.08887691893347698,
+      "grad_norm": 0.5182551741600037,
+      "learning_rate": 2e-05,
+      "loss": 1.0442,
+      "step": 990
+    },
+    {
+      "epoch": 0.08977466558937068,
+      "grad_norm": 0.5045287609100342,
+      "learning_rate": 2e-05,
+      "loss": 1.0133,
+      "step": 1000
+    },
+    {
+      "epoch": 0.09067241224526439,
+      "grad_norm": 0.47217923402786255,
+      "learning_rate": 2e-05,
+      "loss": 1.0102,
+      "step": 1010
+    },
+    {
+      "epoch": 0.09157015890115809,
+      "grad_norm": 0.6045490503311157,
+      "learning_rate": 2e-05,
+      "loss": 1.0759,
+      "step": 1020
+    },
+    {
+      "epoch": 0.0924679055570518,
+      "grad_norm": 0.49416762590408325,
+      "learning_rate": 2e-05,
+      "loss": 0.9857,
+      "step": 1030
+    },
+    {
+      "epoch": 0.0933656522129455,
+      "grad_norm": 0.9547766447067261,
+      "learning_rate": 2e-05,
+      "loss": 1.1359,
+      "step": 1040
+    },
+    {
+      "epoch": 0.09426339886883921,
+      "grad_norm": 0.5142229795455933,
+      "learning_rate": 2e-05,
+      "loss": 1.0329,
+      "step": 1050
+    },
+    {
+      "epoch": 0.09516114552473293,
+      "grad_norm": 0.5049414038658142,
+      "learning_rate": 2e-05,
+      "loss": 0.9621,
+      "step": 1060
+    },
+    {
+      "epoch": 0.09605889218062663,
+      "grad_norm": 0.5428811311721802,
+      "learning_rate": 2e-05,
+      "loss": 1.0414,
+      "step": 1070
+    },
+    {
+      "epoch": 0.09695663883652034,
+      "grad_norm": 0.5135433077812195,
+      "learning_rate": 2e-05,
+      "loss": 1.0206,
+      "step": 1080
+    },
+    {
+      "epoch": 0.09785438549241404,
+      "grad_norm": 0.47476571798324585,
+      "learning_rate": 2e-05,
+      "loss": 1.0308,
+      "step": 1090
+    },
+    {
+      "epoch": 0.09875213214830775,
+      "grad_norm": 0.748688280582428,
+      "learning_rate": 2e-05,
+      "loss": 1.0606,
+      "step": 1100
+    },
+    {
+      "epoch": 0.09964987880420145,
+      "grad_norm": 0.45084625482559204,
+      "learning_rate": 2e-05,
+      "loss": 1.0314,
+      "step": 1110
+    },
+    {
+      "epoch": 0.10054762546009516,
+      "grad_norm": 0.5020058155059814,
+      "learning_rate": 2e-05,
+      "loss": 1.0001,
+      "step": 1120
+    },
+    {
+      "epoch": 0.10144537211598886,
+      "grad_norm": 0.7483564019203186,
+      "learning_rate": 2e-05,
+      "loss": 1.0236,
+      "step": 1130
+    },
+    {
+      "epoch": 0.10234311877188257,
+      "grad_norm": 0.5015250444412231,
+      "learning_rate": 2e-05,
+      "loss": 1.0947,
+      "step": 1140
+    },
+    {
+      "epoch": 0.10324086542777629,
+      "grad_norm": 0.4564529061317444,
+      "learning_rate": 2e-05,
+      "loss": 1.0038,
+      "step": 1150
+    },
+    {
+      "epoch": 0.10413861208366999,
+      "grad_norm": 0.47453996539115906,
+      "learning_rate": 2e-05,
+      "loss": 0.9863,
+      "step": 1160
+    },
+    {
+      "epoch": 0.1050363587395637,
+      "grad_norm": 0.4623263478279114,
+      "learning_rate": 2e-05,
+      "loss": 1.0008,
+      "step": 1170
+    },
+    {
+      "epoch": 0.1059341053954574,
+      "grad_norm": 0.4754187762737274,
+      "learning_rate": 2e-05,
+      "loss": 0.9987,
+      "step": 1180
+    },
+    {
+      "epoch": 0.10683185205135111,
+      "grad_norm": 0.4644615650177002,
+      "learning_rate": 2e-05,
+      "loss": 0.9815,
+      "step": 1190
+    },
+    {
+      "epoch": 0.10772959870724481,
+      "grad_norm": 0.482634037733078,
+      "learning_rate": 2e-05,
+      "loss": 0.9772,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10862734536313852,
+      "grad_norm": 0.5518064498901367,
+      "learning_rate": 2e-05,
+      "loss": 1.0926,
+      "step": 1210
+    },
+    {
+      "epoch": 0.10952509201903222,
+      "grad_norm": 0.5161309242248535,
+      "learning_rate": 2e-05,
+      "loss": 1.0023,
+      "step": 1220
+    },
+    {
+      "epoch": 0.11042283867492593,
+      "grad_norm": 0.47377634048461914,
+      "learning_rate": 2e-05,
+      "loss": 1.0061,
+      "step": 1230
+    },
+    {
+      "epoch": 0.11132058533081965,
+      "grad_norm": 0.5026335716247559,
+      "learning_rate": 2e-05,
+      "loss": 0.9962,
+      "step": 1240
+    },
+    {
+      "epoch": 0.11221833198671335,
+      "grad_norm": 0.5823997855186462,
+      "learning_rate": 2e-05,
+      "loss": 1.0312,
+      "step": 1250
+    },
+    {
+      "epoch": 0.11311607864260706,
+      "grad_norm": 0.44196802377700806,
+      "learning_rate": 2e-05,
+      "loss": 1.0227,
+      "step": 1260
+    },
+    {
+      "epoch": 0.11401382529850076,
+      "grad_norm": 0.48227500915527344,
+      "learning_rate": 2e-05,
+      "loss": 1.041,
+      "step": 1270
+    },
+    {
+      "epoch": 0.11491157195439447,
+      "grad_norm": 0.48953211307525635,
+      "learning_rate": 2e-05,
+      "loss": 1.0208,
+      "step": 1280
+    },
+    {
+      "epoch": 0.11580931861028818,
+      "grad_norm": 0.4923247992992401,
+      "learning_rate": 2e-05,
+      "loss": 1.0013,
+      "step": 1290
+    },
+    {
+      "epoch": 0.11670706526618188,
+      "grad_norm": 0.4701191186904907,
+      "learning_rate": 2e-05,
+      "loss": 1.0063,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11760481192207559,
+      "grad_norm": 0.5244788527488708,
+      "learning_rate": 2e-05,
+      "loss": 0.9743,
+      "step": 1310
+    },
+    {
+      "epoch": 0.11850255857796929,
+      "grad_norm": 0.5617148280143738,
+      "learning_rate": 2e-05,
+      "loss": 1.1172,
+      "step": 1320
+    },
+    {
+      "epoch": 0.11940030523386301,
+      "grad_norm": 0.49794480204582214,
+      "learning_rate": 2e-05,
+      "loss": 0.9922,
+      "step": 1330
+    },
+    {
+      "epoch": 0.12029805188975672,
+      "grad_norm": 0.48185423016548157,
+      "learning_rate": 2e-05,
+      "loss": 0.966,
+      "step": 1340
+    },
+    {
+      "epoch": 0.12119579854565042,
+      "grad_norm": 0.4720144271850586,
+      "learning_rate": 2e-05,
+      "loss": 0.9698,
+      "step": 1350
+    },
+    {
+      "epoch": 0.12209354520154413,
+      "grad_norm": 0.4971640408039093,
+      "learning_rate": 2e-05,
+      "loss": 0.9506,
+      "step": 1360
+    },
+    {
+      "epoch": 0.12299129185743783,
+      "grad_norm": 0.582372784614563,
+      "learning_rate": 2e-05,
+      "loss": 0.9695,
+      "step": 1370
+    },
+    {
+      "epoch": 0.12388903851333154,
+      "grad_norm": 0.5824956297874451,
+      "learning_rate": 2e-05,
+      "loss": 0.9911,
+      "step": 1380
+    },
+    {
+      "epoch": 0.12478678516922524,
+      "grad_norm": 0.49514421820640564,
+      "learning_rate": 2e-05,
+      "loss": 0.9874,
+      "step": 1390
+    },
+    {
+      "epoch": 0.12568453182511896,
+      "grad_norm": 0.520560622215271,
+      "learning_rate": 2e-05,
+      "loss": 1.0279,
+      "step": 1400
+    },
+    {
+      "epoch": 0.12658227848101267,
+      "grad_norm": 0.5677764415740967,
+      "learning_rate": 2e-05,
+      "loss": 1.0537,
+      "step": 1410
+    },
+    {
+      "epoch": 0.12748002513690637,
+      "grad_norm": 0.5025660991668701,
+      "learning_rate": 2e-05,
+      "loss": 0.978,
+      "step": 1420
+    },
+    {
+      "epoch": 0.12837777179280008,
+      "grad_norm": 0.4592083990573883,
+      "learning_rate": 2e-05,
+      "loss": 0.9955,
+      "step": 1430
+    },
+    {
+      "epoch": 0.12927551844869378,
+      "grad_norm": 0.4699345529079437,
+      "learning_rate": 2e-05,
+      "loss": 0.9694,
+      "step": 1440
+    },
+    {
+      "epoch": 0.1301732651045875,
+      "grad_norm": 0.48181068897247314,
+      "learning_rate": 2e-05,
+      "loss": 0.9672,
+      "step": 1450
+    },
+    {
+      "epoch": 0.1310710117604812,
+      "grad_norm": 2.121140956878662,
+      "learning_rate": 2e-05,
+      "loss": 0.9885,
+      "step": 1460
+    },
+    {
+      "epoch": 0.1319687584163749,
+      "grad_norm": 0.47388073801994324,
+      "learning_rate": 2e-05,
+      "loss": 1.0243,
+      "step": 1470
+    },
+    {
+      "epoch": 0.1328665050722686,
+      "grad_norm": 0.6105418801307678,
+      "learning_rate": 2e-05,
+      "loss": 1.024,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1337642517281623,
+      "grad_norm": 0.5576895475387573,
+      "learning_rate": 2e-05,
+      "loss": 0.9856,
+      "step": 1490
+    },
+    {
+      "epoch": 0.13466199838405601,
+      "grad_norm": 0.43749862909317017,
+      "learning_rate": 2e-05,
+      "loss": 0.9399,
+      "step": 1500
+    },
+    {
+      "epoch": 0.13555974503994972,
+      "grad_norm": 0.5869466066360474,
+      "learning_rate": 2e-05,
+      "loss": 1.0006,
+      "step": 1510
+    },
+    {
+      "epoch": 0.13645749169584342,
+      "grad_norm": 0.5816555023193359,
+      "learning_rate": 2e-05,
+      "loss": 0.9505,
+      "step": 1520
+    },
+    {
+      "epoch": 0.13735523835173713,
+      "grad_norm": 0.4605148732662201,
+      "learning_rate": 2e-05,
+      "loss": 1.0417,
+      "step": 1530
+    },
+    {
+      "epoch": 0.13825298500763084,
+      "grad_norm": 0.5583087801933289,
+      "learning_rate": 2e-05,
+      "loss": 1.0041,
+      "step": 1540
+    },
+    {
+      "epoch": 0.13915073166352454,
+      "grad_norm": 0.48093125224113464,
+      "learning_rate": 2e-05,
+      "loss": 0.9559,
+      "step": 1550
+    },
+    {
+      "epoch": 0.14004847831941827,
+      "grad_norm": 0.5225452184677124,
+      "learning_rate": 2e-05,
+      "loss": 0.9674,
+      "step": 1560
+    },
+    {
+      "epoch": 0.14094622497531198,
+      "grad_norm": 0.4931219220161438,
+      "learning_rate": 2e-05,
+      "loss": 1.032,
+      "step": 1570
+    },
+    {
+      "epoch": 0.14184397163120568,
+      "grad_norm": 0.4754665791988373,
+      "learning_rate": 2e-05,
+      "loss": 0.9832,
+      "step": 1580
+    },
+    {
+      "epoch": 0.1427417182870994,
+      "grad_norm": 0.5914980173110962,
+      "learning_rate": 2e-05,
+      "loss": 0.9563,
+      "step": 1590
+    },
+    {
+      "epoch": 0.1436394649429931,
+      "grad_norm": 0.4562107026576996,
+      "learning_rate": 2e-05,
+      "loss": 1.0162,
+      "step": 1600
+    },
+    {
+      "epoch": 0.1445372115988868,
+      "grad_norm": 0.4780389070510864,
+      "learning_rate": 2e-05,
+      "loss": 0.954,
+      "step": 1610
+    },
+    {
+      "epoch": 0.1454349582547805,
+      "grad_norm": 0.8750007748603821,
+      "learning_rate": 2e-05,
+      "loss": 0.9849,
+      "step": 1620
+    },
+    {
+      "epoch": 0.1463327049106742,
+      "grad_norm": 0.48428502678871155,
+      "learning_rate": 2e-05,
+      "loss": 1.0274,
+      "step": 1630
+    },
+    {
+      "epoch": 0.14723045156656792,
+      "grad_norm": 0.4305723309516907,
+      "learning_rate": 2e-05,
+      "loss": 0.9729,
+      "step": 1640
+    },
+    {
+      "epoch": 0.14812819822246162,
+      "grad_norm": 0.5047057867050171,
+      "learning_rate": 2e-05,
+      "loss": 0.997,
+      "step": 1650
+    },
+    {
+      "epoch": 0.14902594487835533,
+      "grad_norm": 0.5030443072319031,
+      "learning_rate": 2e-05,
+      "loss": 1.0052,
+      "step": 1660
+    },
+    {
+      "epoch": 0.14992369153424903,
+      "grad_norm": 0.6058238744735718,
+      "learning_rate": 2e-05,
+      "loss": 0.9561,
+      "step": 1670
+    },
+    {
+      "epoch": 0.15082143819014274,
+      "grad_norm": 0.4600324034690857,
+      "learning_rate": 2e-05,
+      "loss": 0.9747,
+      "step": 1680
+    },
+    {
+      "epoch": 0.15171918484603644,
+      "grad_norm": 0.5196220278739929,
+      "learning_rate": 2e-05,
+      "loss": 0.9739,
+      "step": 1690
+    },
+    {
+      "epoch": 0.15261693150193015,
+      "grad_norm": 0.5066395401954651,
+      "learning_rate": 2e-05,
+      "loss": 0.9805,
+      "step": 1700
+    },
+    {
+      "epoch": 0.15351467815782385,
+      "grad_norm": 0.4423527121543884,
+      "learning_rate": 2e-05,
+      "loss": 1.0143,
+      "step": 1710
+    },
+    {
+      "epoch": 0.15441242481371756,
+      "grad_norm": 0.484889954328537,
+      "learning_rate": 2e-05,
+      "loss": 1.0116,
+      "step": 1720
+    },
+    {
+      "epoch": 0.15531017146961126,
+      "grad_norm": 0.4426444172859192,
+      "learning_rate": 2e-05,
+      "loss": 0.9874,
+      "step": 1730
+    },
+    {
+      "epoch": 0.15620791812550497,
+      "grad_norm": 0.48062387108802795,
+      "learning_rate": 2e-05,
+      "loss": 1.016,
+      "step": 1740
+    },
+    {
+      "epoch": 0.1571056647813987,
+      "grad_norm": 0.5096336603164673,
+      "learning_rate": 2e-05,
+      "loss": 0.9822,
+      "step": 1750
+    },
+    {
+      "epoch": 0.1580034114372924,
+      "grad_norm": 0.5110201835632324,
+      "learning_rate": 2e-05,
+      "loss": 0.9701,
+      "step": 1760
+    },
+    {
+      "epoch": 0.1589011580931861,
+      "grad_norm": 0.5047339200973511,
+      "learning_rate": 2e-05,
+      "loss": 1.0506,
+      "step": 1770
+    },
+    {
+      "epoch": 0.15979890474907982,
+      "grad_norm": 0.5041124820709229,
+      "learning_rate": 2e-05,
+      "loss": 0.9917,
+      "step": 1780
+    },
+    {
+      "epoch": 0.16069665140497352,
+      "grad_norm": 0.5139462947845459,
+      "learning_rate": 2e-05,
+      "loss": 1.0379,
+      "step": 1790
+    },
+    {
+      "epoch": 0.16159439806086723,
+      "grad_norm": 0.45225703716278076,
+      "learning_rate": 2e-05,
+      "loss": 0.994,
+      "step": 1800
+    },
+    {
+      "epoch": 0.16249214471676093,
+      "grad_norm": 0.48371419310569763,
+      "learning_rate": 2e-05,
+      "loss": 0.9789,
+      "step": 1810
+    },
+    {
+      "epoch": 0.16338989137265464,
+      "grad_norm": 1.0779194831848145,
+      "learning_rate": 2e-05,
+      "loss": 1.009,
+      "step": 1820
+    },
+    {
+      "epoch": 0.16428763802854834,
+      "grad_norm": 0.5898749232292175,
+      "learning_rate": 2e-05,
+      "loss": 1.0905,
+      "step": 1830
+    },
+    {
+      "epoch": 0.16518538468444205,
+      "grad_norm": 0.5248507857322693,
+      "learning_rate": 2e-05,
+      "loss": 0.9871,
+      "step": 1840
+    },
+    {
+      "epoch": 0.16608313134033575,
+      "grad_norm": 0.5213241577148438,
+      "learning_rate": 2e-05,
+      "loss": 1.0232,
+      "step": 1850
+    },
+    {
+      "epoch": 0.16698087799622946,
+      "grad_norm": 0.5059240460395813,
+      "learning_rate": 2e-05,
+      "loss": 0.941,
+      "step": 1860
+    },
+    {
+      "epoch": 0.16787862465212317,
+      "grad_norm": 0.49990788102149963,
+      "learning_rate": 2e-05,
+      "loss": 0.9661,
+      "step": 1870
+    },
+    {
+      "epoch": 0.16877637130801687,
+      "grad_norm": 0.5290054678916931,
+      "learning_rate": 2e-05,
+      "loss": 0.9759,
+      "step": 1880
+    },
+    {
+      "epoch": 0.16967411796391058,
+      "grad_norm": 0.48694655299186707,
+      "learning_rate": 2e-05,
+      "loss": 0.9813,
+      "step": 1890
+    },
+    {
+      "epoch": 0.17057186461980428,
+      "grad_norm": 0.7233108878135681,
+      "learning_rate": 2e-05,
+      "loss": 0.9737,
+      "step": 1900
+    },
+    {
+      "epoch": 0.171469611275698,
+      "grad_norm": 0.496572881937027,
+      "learning_rate": 2e-05,
+      "loss": 0.9429,
+      "step": 1910
+    },
+    {
+      "epoch": 0.1723673579315917,
+      "grad_norm": 0.4263172745704651,
+      "learning_rate": 2e-05,
+      "loss": 0.9352,
+      "step": 1920
+    },
+    {
+      "epoch": 0.17326510458748542,
+      "grad_norm": 0.4787362217903137,
+      "learning_rate": 2e-05,
+      "loss": 0.9401,
+      "step": 1930
+    },
+    {
+      "epoch": 0.17416285124337913,
+      "grad_norm": 0.46687498688697815,
+      "learning_rate": 2e-05,
+      "loss": 0.96,
+      "step": 1940
+    },
+    {
+      "epoch": 0.17506059789927284,
+      "grad_norm": 0.5431246757507324,
+      "learning_rate": 2e-05,
+      "loss": 0.9933,
+      "step": 1950
+    },
+    {
+      "epoch": 0.17595834455516654,
+      "grad_norm": 0.5195553302764893,
+      "learning_rate": 2e-05,
+      "loss": 0.9357,
+      "step": 1960
+    },
+    {
+      "epoch": 0.17685609121106025,
+      "grad_norm": 0.47831055521965027,
+      "learning_rate": 2e-05,
+      "loss": 0.9931,
+      "step": 1970
+    },
+    {
+      "epoch": 0.17775383786695395,
+      "grad_norm": 0.4682317078113556,
+      "learning_rate": 2e-05,
+      "loss": 0.926,
+      "step": 1980
+    },
+    {
+      "epoch": 0.17865158452284766,
+      "grad_norm": 0.5072707533836365,
+      "learning_rate": 2e-05,
+      "loss": 0.9792,
+      "step": 1990
+    },
+    {
+      "epoch": 0.17954933117874136,
+      "grad_norm": 0.5063421130180359,
+      "learning_rate": 2e-05,
+      "loss": 0.9729,
+      "step": 2000
+    },
+    {
+      "epoch": 0.18044707783463507,
+      "grad_norm": 0.47064459323883057,
+      "learning_rate": 2e-05,
+      "loss": 0.9511,
+      "step": 2010
+    },
+    {
+      "epoch": 0.18134482449052877,
+      "grad_norm": 0.48214080929756165,
+      "learning_rate": 2e-05,
+      "loss": 0.9992,
+      "step": 2020
+    },
+    {
+      "epoch": 0.18224257114642248,
+      "grad_norm": 0.48727652430534363,
+      "learning_rate": 2e-05,
+      "loss": 0.9414,
+      "step": 2030
+    },
+    {
+      "epoch": 0.18314031780231618,
+      "grad_norm": 0.5159448981285095,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2040
+    },
+    {
+      "epoch": 0.1840380644582099,
+      "grad_norm": 0.4612741768360138,
+      "learning_rate": 2e-05,
+      "loss": 0.9929,
+      "step": 2050
+    },
+    {
+      "epoch": 0.1849358111141036,
+      "grad_norm": 0.4544714689254761,
+      "learning_rate": 2e-05,
+      "loss": 0.9007,
+      "step": 2060
+    },
+    {
+      "epoch": 0.1858335577699973,
+      "grad_norm": 0.4296947419643402,
+      "learning_rate": 2e-05,
+      "loss": 0.9784,
+      "step": 2070
+    },
+    {
+      "epoch": 0.186731304425891,
+      "grad_norm": 0.455415815114975,
+      "learning_rate": 2e-05,
+      "loss": 0.9248,
+      "step": 2080
+    },
+    {
+      "epoch": 0.1876290510817847,
+      "grad_norm": 0.4618610441684723,
+      "learning_rate": 2e-05,
+      "loss": 0.9991,
+      "step": 2090
+    },
+    {
+      "epoch": 0.18852679773767841,
+      "grad_norm": 0.5599287152290344,
+      "learning_rate": 2e-05,
+      "loss": 0.9861,
+      "step": 2100
+    },
+    {
+      "epoch": 0.18942454439357215,
+      "grad_norm": 0.49464237689971924,
+      "learning_rate": 2e-05,
+      "loss": 1.0128,
+      "step": 2110
+    },
+    {
+      "epoch": 0.19032229104946585,
+      "grad_norm": 0.5259017944335938,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2120
+    },
+    {
+      "epoch": 0.19122003770535956,
+      "grad_norm": 0.4654732942581177,
+      "learning_rate": 2e-05,
+      "loss": 1.0427,
+      "step": 2130
+    },
+    {
+      "epoch": 0.19211778436125326,
+      "grad_norm": 0.7427297234535217,
+      "learning_rate": 2e-05,
+      "loss": 0.953,
+      "step": 2140
+    },
+    {
+      "epoch": 0.19301553101714697,
+      "grad_norm": 0.47707414627075195,
+      "learning_rate": 2e-05,
+      "loss": 0.956,
+      "step": 2150
+    },
+    {
+      "epoch": 0.19391327767304067,
+      "grad_norm": 0.4793659448623657,
+      "learning_rate": 2e-05,
+      "loss": 0.9417,
+      "step": 2160
+    },
+    {
+      "epoch": 0.19481102432893438,
+      "grad_norm": 0.5135785341262817,
+      "learning_rate": 2e-05,
+      "loss": 0.9752,
+      "step": 2170
+    },
+    {
+      "epoch": 0.19570877098482808,
+      "grad_norm": 0.5346530079841614,
+      "learning_rate": 2e-05,
+      "loss": 0.9438,
+      "step": 2180
+    },
+    {
+      "epoch": 0.1966065176407218,
+      "grad_norm": 0.4482640027999878,
+      "learning_rate": 2e-05,
+      "loss": 1.005,
+      "step": 2190
+    },
+    {
+      "epoch": 0.1975042642966155,
+      "grad_norm": 0.46668392419815063,
+      "learning_rate": 2e-05,
+      "loss": 0.9477,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1984020109525092,
+      "grad_norm": 0.46946898102760315,
+      "learning_rate": 2e-05,
+      "loss": 0.9564,
+      "step": 2210
+    },
+    {
+      "epoch": 0.1992997576084029,
+      "grad_norm": 0.5270060896873474,
+      "learning_rate": 2e-05,
+      "loss": 0.9895,
+      "step": 2220
+    },
+    {
+      "epoch": 0.2001975042642966,
+      "grad_norm": 0.5055740475654602,
+      "learning_rate": 2e-05,
+      "loss": 0.8949,
+      "step": 2230
+    },
+    {
+      "epoch": 0.20109525092019032,
+      "grad_norm": 0.4616740047931671,
+      "learning_rate": 2e-05,
+      "loss": 0.97,
+      "step": 2240
+    },
+    {
+      "epoch": 0.20199299757608402,
+      "grad_norm": 0.4541808068752289,
+      "learning_rate": 2e-05,
+      "loss": 0.9442,
+      "step": 2250
+    },
+    {
+      "epoch": 0.20289074423197773,
+      "grad_norm": 0.5195493698120117,
+      "learning_rate": 2e-05,
+      "loss": 0.9948,
+      "step": 2260
+    },
+    {
+      "epoch": 0.20378849088787143,
+      "grad_norm": 0.5284701585769653,
+      "learning_rate": 2e-05,
+      "loss": 0.9796,
+      "step": 2270
+    },
+    {
+      "epoch": 0.20468623754376514,
+      "grad_norm": 0.47338250279426575,
+      "learning_rate": 2e-05,
+      "loss": 0.9378,
+      "step": 2280
+    },
+    {
+      "epoch": 0.20558398419965884,
+      "grad_norm": 0.4906748831272125,
+      "learning_rate": 2e-05,
+      "loss": 0.9891,
+      "step": 2290
+    },
+    {
+      "epoch": 0.20648173085555258,
+      "grad_norm": 0.4232115149497986,
+      "learning_rate": 2e-05,
+      "loss": 0.9159,
+      "step": 2300
+    },
+    {
+      "epoch": 0.20737947751144628,
+      "grad_norm": 0.4847649037837982,
+      "learning_rate": 2e-05,
+      "loss": 0.9513,
+      "step": 2310
+    },
+    {
+      "epoch": 0.20827722416733999,
+      "grad_norm": 0.5703267455101013,
+      "learning_rate": 2e-05,
+      "loss": 0.9478,
+      "step": 2320
+    },
+    {
+      "epoch": 0.2091749708232337,
+      "grad_norm": 0.48311203718185425,
+      "learning_rate": 2e-05,
+      "loss": 0.935,
+      "step": 2330
+    },
+    {
+      "epoch": 0.2100727174791274,
+      "grad_norm": 0.6760455369949341,
+      "learning_rate": 2e-05,
+      "loss": 0.9143,
+      "step": 2340
+    },
+    {
+      "epoch": 0.2109704641350211,
+      "grad_norm": 0.48834580183029175,
+      "learning_rate": 2e-05,
+      "loss": 0.9758,
+      "step": 2350
+    },
+    {
+      "epoch": 0.2118682107909148,
+      "grad_norm": 0.5034499764442444,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2360
+    },
+    {
+      "epoch": 0.2127659574468085,
+      "grad_norm": 0.47994714975357056,
+      "learning_rate": 2e-05,
+      "loss": 0.9517,
+      "step": 2370
+    },
+    {
+      "epoch": 0.21366370410270222,
+      "grad_norm": 0.42571839690208435,
+      "learning_rate": 2e-05,
+      "loss": 0.9645,
+      "step": 2380
+    },
+    {
+      "epoch": 0.21456145075859592,
+      "grad_norm": 0.49876219034194946,
+      "learning_rate": 2e-05,
+      "loss": 0.9437,
+      "step": 2390
+    },
+    {
+      "epoch": 0.21545919741448963,
+      "grad_norm": 0.46612128615379333,
+      "learning_rate": 2e-05,
+      "loss": 0.9414,
+      "step": 2400
+    },
+    {
+      "epoch": 0.21635694407038333,
+      "grad_norm": 0.4798942804336548,
+      "learning_rate": 2e-05,
+      "loss": 0.8928,
+      "step": 2410
+    },
+    {
+      "epoch": 0.21725469072627704,
+      "grad_norm": 0.4814785420894623,
+      "learning_rate": 2e-05,
+      "loss": 0.957,
+      "step": 2420
+    },
+    {
+      "epoch": 0.21815243738217074,
+      "grad_norm": 0.45460546016693115,
+      "learning_rate": 2e-05,
+      "loss": 0.9066,
+      "step": 2430
+    },
+    {
+      "epoch": 0.21905018403806445,
+      "grad_norm": 0.4495159387588501,
+      "learning_rate": 2e-05,
+      "loss": 0.9417,
+      "step": 2440
+    },
+    {
+      "epoch": 0.21994793069395815,
+      "grad_norm": 0.5322041511535645,
+      "learning_rate": 2e-05,
+      "loss": 0.9186,
+      "step": 2450
+    },
+    {
+      "epoch": 0.22084567734985186,
+      "grad_norm": 0.4809408187866211,
+      "learning_rate": 2e-05,
+      "loss": 0.9293,
+      "step": 2460
+    },
+    {
+      "epoch": 0.22174342400574557,
+      "grad_norm": 0.48313039541244507,
+      "learning_rate": 2e-05,
+      "loss": 0.9358,
+      "step": 2470
+    },
+    {
+      "epoch": 0.2226411706616393,
+      "grad_norm": 0.5264012217521667,
+      "learning_rate": 2e-05,
+      "loss": 0.9699,
+      "step": 2480
+    },
+    {
+      "epoch": 0.223538917317533,
+      "grad_norm": 0.4506959021091461,
+      "learning_rate": 2e-05,
+      "loss": 0.9113,
+      "step": 2490
+    },
+    {
+      "epoch": 0.2244366639734267,
+      "grad_norm": 0.5064295530319214,
+      "learning_rate": 2e-05,
+      "loss": 0.9801,
+      "step": 2500
+    },
+    {
+      "epoch": 0.22533441062932041,
+      "grad_norm": 3.251514196395874,
+      "learning_rate": 2e-05,
+      "loss": 0.9402,
+      "step": 2510
+    },
+    {
+      "epoch": 0.22623215728521412,
+      "grad_norm": 0.6295604109764099,
+      "learning_rate": 2e-05,
+      "loss": 0.9179,
+      "step": 2520
+    },
+    {
+      "epoch": 0.22712990394110782,
+      "grad_norm": 0.45736199617385864,
+      "learning_rate": 2e-05,
+      "loss": 0.8863,
+      "step": 2530
+    },
+    {
+      "epoch": 0.22802765059700153,
+      "grad_norm": 0.5029098987579346,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2540
+    },
+    {
+      "epoch": 0.22892539725289524,
+      "grad_norm": 0.46901586651802063,
+      "learning_rate": 2e-05,
+      "loss": 0.9845,
+      "step": 2550
+    },
+    {
+      "epoch": 0.22982314390878894,
+      "grad_norm": 0.4977826774120331,
+      "learning_rate": 2e-05,
+      "loss": 0.9733,
+      "step": 2560
+    },
+    {
+      "epoch": 0.23072089056468265,
+      "grad_norm": 0.45413094758987427,
+      "learning_rate": 2e-05,
+      "loss": 0.9495,
+      "step": 2570
+    },
+    {
+      "epoch": 0.23161863722057635,
+      "grad_norm": 0.5607425570487976,
+      "learning_rate": 2e-05,
+      "loss": 0.996,
+      "step": 2580
+    },
+    {
+      "epoch": 0.23251638387647006,
+      "grad_norm": 0.5477047562599182,
+      "learning_rate": 2e-05,
+      "loss": 0.9223,
+      "step": 2590
+    },
+    {
+      "epoch": 0.23341413053236376,
+      "grad_norm": 0.44483038783073425,
+      "learning_rate": 2e-05,
+      "loss": 0.9252,
+      "step": 2600
+    },
+    {
+      "epoch": 0.23431187718825747,
+      "grad_norm": 0.4706748127937317,
+      "learning_rate": 2e-05,
+      "loss": 0.9116,
+      "step": 2610
+    },
+    {
+      "epoch": 0.23520962384415117,
+      "grad_norm": 0.4670526385307312,
+      "learning_rate": 2e-05,
+      "loss": 0.9261,
+      "step": 2620
+    },
+    {
+      "epoch": 0.23610737050004488,
+      "grad_norm": 0.449673593044281,
+      "learning_rate": 2e-05,
+      "loss": 0.8981,
+      "step": 2630
+    },
+    {
+      "epoch": 0.23700511715593858,
+      "grad_norm": 0.4667699933052063,
+      "learning_rate": 2e-05,
+      "loss": 0.9685,
+      "step": 2640
+    },
+    {
+      "epoch": 0.2379028638118323,
+      "grad_norm": 0.5052064657211304,
+      "learning_rate": 2e-05,
+      "loss": 0.8955,
+      "step": 2650
+    },
+    {
+      "epoch": 0.23880061046772602,
+      "grad_norm": 0.4791005551815033,
+      "learning_rate": 2e-05,
+      "loss": 0.9012,
+      "step": 2660
+    },
+    {
+      "epoch": 0.23969835712361973,
+      "grad_norm": 0.44844916462898254,
+      "learning_rate": 2e-05,
+      "loss": 0.8609,
+      "step": 2670
+    },
+    {
+      "epoch": 0.24059610377951343,
+      "grad_norm": 2.4957547187805176,
+      "learning_rate": 2e-05,
+      "loss": 0.9133,
+      "step": 2680
+    },
+    {
+      "epoch": 0.24149385043540714,
+      "grad_norm": 0.5713205337524414,
+      "learning_rate": 2e-05,
+      "loss": 0.9941,
+      "step": 2690
+    },
+    {
+      "epoch": 0.24239159709130084,
+      "grad_norm": 0.48091888427734375,
+      "learning_rate": 2e-05,
+      "loss": 0.9192,
+      "step": 2700
+    },
+    {
+      "epoch": 0.24328934374719455,
+      "grad_norm": 0.5733251571655273,
+      "learning_rate": 2e-05,
+      "loss": 0.98,
+      "step": 2710
+    },
+    {
+      "epoch": 0.24418709040308825,
+      "grad_norm": 0.48650702834129333,
+      "learning_rate": 2e-05,
+      "loss": 0.9702,
+      "step": 2720
+    },
+    {
+      "epoch": 0.24508483705898196,
+      "grad_norm": 0.5046181678771973,
+      "learning_rate": 2e-05,
+      "loss": 0.9079,
+      "step": 2730
+    },
+    {
+      "epoch": 0.24598258371487566,
+      "grad_norm": 0.42475616931915283,
+      "learning_rate": 2e-05,
+      "loss": 0.9127,
+      "step": 2740
+    },
+    {
+      "epoch": 0.24688033037076937,
+      "grad_norm": 0.6360783576965332,
+      "learning_rate": 2e-05,
+      "loss": 0.9374,
+      "step": 2750
+    },
+    {
+      "epoch": 0.24777807702666307,
+      "grad_norm": 0.4691959023475647,
+      "learning_rate": 2e-05,
+      "loss": 0.9276,
+      "step": 2760
+    },
+    {
+      "epoch": 0.24867582368255678,
+      "grad_norm": 0.5178162455558777,
+      "learning_rate": 2e-05,
+      "loss": 0.9005,
+      "step": 2770
+    },
+    {
+      "epoch": 0.24957357033845048,
+      "grad_norm": 0.5031567215919495,
+      "learning_rate": 2e-05,
+      "loss": 0.9071,
+      "step": 2780
+    },
+    {
+      "epoch": 0.2504713169943442,
+      "grad_norm": 0.4603200852870941,
+      "learning_rate": 2e-05,
+      "loss": 0.8836,
+      "step": 2790
+    },
+    {
+      "epoch": 0.2513690636502379,
+      "grad_norm": 0.46788638830184937,
+      "learning_rate": 2e-05,
+      "loss": 0.9372,
+      "step": 2800
+    },
+    {
+      "epoch": 0.25226681030613163,
+      "grad_norm": 0.4958333969116211,
+      "learning_rate": 2e-05,
+      "loss": 0.8879,
+      "step": 2810
+    },
+    {
+      "epoch": 0.25316455696202533,
+      "grad_norm": 0.5206777453422546,
+      "learning_rate": 2e-05,
+      "loss": 0.9495,
+      "step": 2820
+    },
+    {
+      "epoch": 0.25406230361791904,
+      "grad_norm": 0.4814874231815338,
+      "learning_rate": 2e-05,
+      "loss": 0.9553,
+      "step": 2830
+    },
+    {
+      "epoch": 0.25496005027381274,
+      "grad_norm": 0.5589430928230286,
+      "learning_rate": 2e-05,
+      "loss": 0.94,
+      "step": 2840
+    },
+    {
+      "epoch": 0.25585779692970645,
+      "grad_norm": 0.4594886898994446,
+      "learning_rate": 2e-05,
+      "loss": 0.9163,
+      "step": 2850
+    },
+    {
+      "epoch": 0.25675554358560015,
+      "grad_norm": 0.4813527464866638,
+      "learning_rate": 2e-05,
+      "loss": 0.9232,
+      "step": 2860
+    },
+    {
+      "epoch": 0.25765329024149386,
+      "grad_norm": 0.5455467104911804,
+      "learning_rate": 2e-05,
+      "loss": 0.8985,
+      "step": 2870
+    },
+    {
+      "epoch": 0.25855103689738757,
+      "grad_norm": 0.4964544475078583,
+      "learning_rate": 2e-05,
+      "loss": 0.9342,
+      "step": 2880
+    },
+    {
+      "epoch": 0.25944878355328127,
+      "grad_norm": 0.5281876921653748,
+      "learning_rate": 2e-05,
+      "loss": 0.9171,
+      "step": 2890
+    },
+    {
+      "epoch": 0.260346530209175,
+      "grad_norm": 0.45764508843421936,
+      "learning_rate": 2e-05,
+      "loss": 0.9298,
+      "step": 2900
+    },
+    {
+      "epoch": 0.2612442768650687,
+      "grad_norm": 0.4511374533176422,
+      "learning_rate": 2e-05,
+      "loss": 0.9597,
+      "step": 2910
+    },
+    {
+      "epoch": 0.2621420235209624,
+      "grad_norm": 0.564020574092865,
+      "learning_rate": 2e-05,
+      "loss": 0.9749,
+      "step": 2920
+    },
+    {
+      "epoch": 0.2630397701768561,
+      "grad_norm": 0.5097768902778625,
+      "learning_rate": 2e-05,
+      "loss": 0.9651,
+      "step": 2930
+    },
+    {
+      "epoch": 0.2639375168327498,
+      "grad_norm": 0.9212029576301575,
+      "learning_rate": 2e-05,
+      "loss": 0.9134,
+      "step": 2940
+    },
+    {
+      "epoch": 0.2648352634886435,
+      "grad_norm": 0.4786180257797241,
+      "learning_rate": 2e-05,
+      "loss": 0.8749,
+      "step": 2950
+    },
+    {
+      "epoch": 0.2657330101445372,
+      "grad_norm": 0.7829163074493408,
+      "learning_rate": 2e-05,
+      "loss": 0.9437,
+      "step": 2960
+    },
+    {
+      "epoch": 0.2666307568004309,
+      "grad_norm": 0.43390172719955444,
+      "learning_rate": 2e-05,
+      "loss": 0.8933,
+      "step": 2970
+    },
+    {
+      "epoch": 0.2675285034563246,
+      "grad_norm": 0.47209569811820984,
+      "learning_rate": 2e-05,
+      "loss": 0.9175,
+      "step": 2980
+    },
+    {
+      "epoch": 0.2684262501122183,
+      "grad_norm": 0.48048534989356995,
+      "learning_rate": 2e-05,
+      "loss": 0.8683,
+      "step": 2990
+    },
+    {
+      "epoch": 0.26932399676811203,
+      "grad_norm": 0.4526227116584778,
+      "learning_rate": 2e-05,
+      "loss": 0.9141,
+      "step": 3000
+    },
+    {
+      "epoch": 0.27022174342400573,
+      "grad_norm": 0.5780067443847656,
+      "learning_rate": 2e-05,
+      "loss": 0.9579,
+      "step": 3010
+    },
+    {
+      "epoch": 0.27111949007989944,
+      "grad_norm": 0.4939303398132324,
+      "learning_rate": 2e-05,
+      "loss": 0.9292,
+      "step": 3020
+    },
+    {
+      "epoch": 0.27201723673579314,
+      "grad_norm": 0.48181208968162537,
+      "learning_rate": 2e-05,
+      "loss": 0.9364,
+      "step": 3030
+    },
+    {
+      "epoch": 0.27291498339168685,
+      "grad_norm": 0.5183773636817932,
+      "learning_rate": 2e-05,
+      "loss": 0.888,
+      "step": 3040
+    },
+    {
+      "epoch": 0.27381273004758055,
+      "grad_norm": 0.5430471301078796,
+      "learning_rate": 2e-05,
+      "loss": 0.8965,
+      "step": 3050
+    },
+    {
+      "epoch": 0.27471047670347426,
+      "grad_norm": 0.43764039874076843,
+      "learning_rate": 2e-05,
+      "loss": 0.9062,
+      "step": 3060
+    },
+    {
+      "epoch": 0.27560822335936797,
+      "grad_norm": 0.5781585574150085,
+      "learning_rate": 2e-05,
+      "loss": 0.8996,
+      "step": 3070
+    },
+    {
+      "epoch": 0.27650597001526167,
+      "grad_norm": 0.4882710874080658,
+      "learning_rate": 2e-05,
+      "loss": 0.962,
+      "step": 3080
+    },
+    {
+      "epoch": 0.2774037166711554,
+      "grad_norm": 0.5440769791603088,
+      "learning_rate": 2e-05,
+      "loss": 0.9125,
+      "step": 3090
+    },
+    {
+      "epoch": 0.2783014633270491,
+      "grad_norm": 0.5098283886909485,
+      "learning_rate": 2e-05,
+      "loss": 0.9041,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2791992099829428,
+      "grad_norm": 0.43088769912719727,
+      "learning_rate": 2e-05,
+      "loss": 0.9121,
+      "step": 3110
+    },
+    {
+      "epoch": 0.28009695663883655,
+      "grad_norm": 0.5979171991348267,
+      "learning_rate": 2e-05,
+      "loss": 0.9265,
+      "step": 3120
+    },
+    {
+      "epoch": 0.28099470329473025,
+      "grad_norm": 0.49221566319465637,
+      "learning_rate": 2e-05,
+      "loss": 0.8858,
+      "step": 3130
+    },
+    {
+      "epoch": 0.28189244995062396,
+      "grad_norm": 0.469924658536911,
+      "learning_rate": 2e-05,
+      "loss": 0.9277,
+      "step": 3140
+    },
+    {
+      "epoch": 0.28279019660651766,
+      "grad_norm": 0.4510713219642639,
+      "learning_rate": 2e-05,
+      "loss": 0.9147,
+      "step": 3150
+    },
+    {
+      "epoch": 0.28368794326241137,
+      "grad_norm": 0.5212750434875488,
+      "learning_rate": 2e-05,
+      "loss": 0.8694,
+      "step": 3160
+    },
+    {
+      "epoch": 0.2845856899183051,
+      "grad_norm": 0.604622483253479,
+      "learning_rate": 2e-05,
+      "loss": 1.0243,
+      "step": 3170
+    },
+    {
+      "epoch": 0.2854834365741988,
+      "grad_norm": 0.5070410966873169,
+      "learning_rate": 2e-05,
+      "loss": 0.9938,
+      "step": 3180
+    },
+    {
+      "epoch": 0.2863811832300925,
+      "grad_norm": 0.47141265869140625,
+      "learning_rate": 2e-05,
+      "loss": 0.9229,
+      "step": 3190
+    },
+    {
+      "epoch": 0.2872789298859862,
+      "grad_norm": 0.49983882904052734,
+      "learning_rate": 2e-05,
+      "loss": 0.9245,
+      "step": 3200
+    },
+    {
+      "epoch": 0.2881766765418799,
+      "grad_norm": 0.44442322850227356,
+      "learning_rate": 2e-05,
+      "loss": 0.8915,
+      "step": 3210
+    },
+    {
+      "epoch": 0.2890744231977736,
+      "grad_norm": 0.5937116146087646,
+      "learning_rate": 2e-05,
+      "loss": 0.8815,
+      "step": 3220
+    },
+    {
+      "epoch": 0.2899721698536673,
+      "grad_norm": 0.46363648772239685,
+      "learning_rate": 2e-05,
+      "loss": 0.9391,
+      "step": 3230
+    },
+    {
+      "epoch": 0.290869916509561,
+      "grad_norm": 0.49822157621383667,
+      "learning_rate": 2e-05,
+      "loss": 0.9411,
+      "step": 3240
+    },
+    {
+      "epoch": 0.2917676631654547,
+      "grad_norm": 0.454148530960083,
+      "learning_rate": 2e-05,
+      "loss": 0.9604,
+      "step": 3250
+    },
+    {
+      "epoch": 0.2926654098213484,
+      "grad_norm": 0.4586525857448578,
+      "learning_rate": 2e-05,
+      "loss": 0.8883,
+      "step": 3260
+    },
+    {
+      "epoch": 0.2935631564772421,
+      "grad_norm": 0.523205578327179,
+      "learning_rate": 2e-05,
+      "loss": 0.9663,
+      "step": 3270
+    },
+    {
+      "epoch": 0.29446090313313583,
+      "grad_norm": 0.4817197322845459,
+      "learning_rate": 2e-05,
+      "loss": 1.0149,
+      "step": 3280
+    },
+    {
+      "epoch": 0.29535864978902954,
+      "grad_norm": 0.4903642237186432,
+      "learning_rate": 2e-05,
+      "loss": 0.8846,
+      "step": 3290
+    },
+    {
+      "epoch": 0.29625639644492324,
+      "grad_norm": 0.614147961139679,
+      "learning_rate": 2e-05,
+      "loss": 0.933,
+      "step": 3300
+    },
+    {
+      "epoch": 0.29715414310081695,
+      "grad_norm": 0.7195588946342468,
+      "learning_rate": 2e-05,
+      "loss": 0.9394,
+      "step": 3310
+    },
+    {
+      "epoch": 0.29805188975671065,
+      "grad_norm": 0.4897194504737854,
+      "learning_rate": 2e-05,
+      "loss": 0.9322,
+      "step": 3320
+    },
+    {
+      "epoch": 0.29894963641260436,
+      "grad_norm": 0.45128145813941956,
+      "learning_rate": 2e-05,
+      "loss": 0.9344,
+      "step": 3330
+    },
+    {
+      "epoch": 0.29984738306849806,
+      "grad_norm": 0.822573721408844,
+      "learning_rate": 2e-05,
+      "loss": 0.9841,
+      "step": 3340
+    },
+    {
+      "epoch": 0.30074512972439177,
+      "grad_norm": 0.5394124388694763,
+      "learning_rate": 2e-05,
+      "loss": 0.9147,
+      "step": 3350
+    },
+    {
+      "epoch": 0.3016428763802855,
+      "grad_norm": 0.49424466490745544,
+      "learning_rate": 2e-05,
+      "loss": 0.9462,
+      "step": 3360
+    },
+    {
+      "epoch": 0.3025406230361792,
+      "grad_norm": 0.42718440294265747,
+      "learning_rate": 2e-05,
+      "loss": 0.8847,
+      "step": 3370
+    },
+    {
+      "epoch": 0.3034383696920729,
+      "grad_norm": 0.46531885862350464,
+      "learning_rate": 2e-05,
+      "loss": 0.9212,
+      "step": 3380
+    },
+    {
+      "epoch": 0.3043361163479666,
+      "grad_norm": 0.48290079832077026,
+      "learning_rate": 2e-05,
+      "loss": 0.883,
+      "step": 3390
+    },
+    {
+      "epoch": 0.3052338630038603,
+      "grad_norm": 0.46699535846710205,
+      "learning_rate": 2e-05,
+      "loss": 0.8793,
+      "step": 3400
+    },
+    {
+      "epoch": 0.306131609659754,
+      "grad_norm": 0.4600708484649658,
+      "learning_rate": 2e-05,
+      "loss": 0.9552,
+      "step": 3410
+    },
+    {
+      "epoch": 0.3070293563156477,
+      "grad_norm": 0.7517644762992859,
+      "learning_rate": 2e-05,
+      "loss": 0.9739,
+      "step": 3420
+    },
+    {
+      "epoch": 0.3079271029715414,
+      "grad_norm": 0.4827682673931122,
+      "learning_rate": 2e-05,
+      "loss": 0.9012,
+      "step": 3430
+    },
+    {
+      "epoch": 0.3088248496274351,
+      "grad_norm": 0.4797245264053345,
+      "learning_rate": 2e-05,
+      "loss": 0.8966,
+      "step": 3440
+    },
+    {
+      "epoch": 0.3097225962833288,
+      "grad_norm": 0.4818043112754822,
+      "learning_rate": 2e-05,
+      "loss": 0.8466,
+      "step": 3450
+    },
+    {
+      "epoch": 0.3106203429392225,
+      "grad_norm": 0.4650554358959198,
+      "learning_rate": 2e-05,
+      "loss": 0.9051,
+      "step": 3460
+    },
+    {
+      "epoch": 0.31151808959511623,
+      "grad_norm": 0.5040600299835205,
+      "learning_rate": 2e-05,
+      "loss": 0.8793,
+      "step": 3470
+    },
+    {
+      "epoch": 0.31241583625100994,
+      "grad_norm": 0.48019030690193176,
+      "learning_rate": 2e-05,
+      "loss": 0.8922,
+      "step": 3480
+    },
+    {
+      "epoch": 0.3133135829069037,
+      "grad_norm": 0.46960967779159546,
+      "learning_rate": 2e-05,
+      "loss": 0.9479,
+      "step": 3490
+    },
+    {
+      "epoch": 0.3142113295627974,
+      "grad_norm": 0.49538564682006836,
+      "learning_rate": 2e-05,
+      "loss": 0.8339,
+      "step": 3500
+    },
+    {
+      "epoch": 0.3151090762186911,
+      "grad_norm": 0.47915852069854736,
+      "learning_rate": 2e-05,
+      "loss": 0.9401,
+      "step": 3510
+    },
+    {
+      "epoch": 0.3160068228745848,
+      "grad_norm": 0.5094796419143677,
+      "learning_rate": 2e-05,
+      "loss": 0.9118,
+      "step": 3520
+    },
+    {
+      "epoch": 0.3169045695304785,
+      "grad_norm": 0.4799898862838745,
+      "learning_rate": 2e-05,
+      "loss": 0.8832,
+      "step": 3530
+    },
+    {
+      "epoch": 0.3178023161863722,
+      "grad_norm": 0.49820947647094727,
+      "learning_rate": 2e-05,
+      "loss": 0.8841,
+      "step": 3540
+    },
+    {
+      "epoch": 0.31870006284226593,
+      "grad_norm": 0.4735852777957916,
+      "learning_rate": 2e-05,
+      "loss": 0.8905,
+      "step": 3550
+    },
+    {
+      "epoch": 0.31959780949815964,
+      "grad_norm": 0.44788357615470886,
+      "learning_rate": 2e-05,
+      "loss": 0.9423,
+      "step": 3560
+    },
+    {
+      "epoch": 0.32049555615405334,
+      "grad_norm": 0.45613691210746765,
+      "learning_rate": 2e-05,
+      "loss": 0.8842,
+      "step": 3570
+    },
+    {
+      "epoch": 0.32139330280994705,
+      "grad_norm": 0.4647972583770752,
+      "learning_rate": 2e-05,
+      "loss": 0.8866,
+      "step": 3580
+    },
+    {
+      "epoch": 0.32229104946584075,
+      "grad_norm": 0.45353108644485474,
+      "learning_rate": 2e-05,
+      "loss": 0.8554,
+      "step": 3590
+    },
+    {
+      "epoch": 0.32318879612173446,
+      "grad_norm": 0.45523348450660706,
+      "learning_rate": 2e-05,
+      "loss": 0.832,
+      "step": 3600
+    },
+    {
+      "epoch": 0.32408654277762816,
+      "grad_norm": 0.48829853534698486,
+      "learning_rate": 2e-05,
+      "loss": 0.9183,
+      "step": 3610
+    },
+    {
+      "epoch": 0.32498428943352187,
+      "grad_norm": 0.47700899839401245,
+      "learning_rate": 2e-05,
+      "loss": 0.8169,
+      "step": 3620
+    },
+    {
+      "epoch": 0.32588203608941557,
+      "grad_norm": 0.437050461769104,
+      "learning_rate": 2e-05,
+      "loss": 0.8984,
+      "step": 3630
+    },
+    {
+      "epoch": 0.3267797827453093,
+      "grad_norm": 0.46260592341423035,
+      "learning_rate": 2e-05,
+      "loss": 0.9305,
+      "step": 3640
+    },
+    {
+      "epoch": 0.327677529401203,
+      "grad_norm": 0.5209080576896667,
+      "learning_rate": 2e-05,
+      "loss": 0.8682,
+      "step": 3650
+    },
+    {
+      "epoch": 0.3285752760570967,
+      "grad_norm": 0.457423597574234,
+      "learning_rate": 2e-05,
+      "loss": 0.8895,
+      "step": 3660
+    },
+    {
+      "epoch": 0.3294730227129904,
+      "grad_norm": 0.4516398012638092,
+      "learning_rate": 2e-05,
+      "loss": 0.888,
+      "step": 3670
+    },
+    {
+      "epoch": 0.3303707693688841,
+      "grad_norm": 0.45212408900260925,
+      "learning_rate": 2e-05,
+      "loss": 0.8447,
+      "step": 3680
+    },
+    {
+      "epoch": 0.3312685160247778,
+      "grad_norm": 0.467307984828949,
+      "learning_rate": 2e-05,
+      "loss": 0.9073,
+      "step": 3690
+    },
+    {
+      "epoch": 0.3321662626806715,
+      "grad_norm": 0.48582276701927185,
+      "learning_rate": 2e-05,
+      "loss": 0.8964,
+      "step": 3700
+    },
+    {
+      "epoch": 0.3330640093365652,
+      "grad_norm": 0.4898392856121063,
+      "learning_rate": 2e-05,
+      "loss": 0.9417,
+      "step": 3710
+    },
+    {
+      "epoch": 0.3339617559924589,
+      "grad_norm": 0.5412060618400574,
+      "learning_rate": 2e-05,
+      "loss": 0.8814,
+      "step": 3720
+    },
+    {
+      "epoch": 0.3348595026483526,
+      "grad_norm": 0.5982813835144043,
+      "learning_rate": 2e-05,
+      "loss": 0.9614,
+      "step": 3730
+    },
+    {
+      "epoch": 0.33575724930424633,
+      "grad_norm": 0.4588804244995117,
+      "learning_rate": 2e-05,
+      "loss": 0.8505,
+      "step": 3740
+    },
+    {
+      "epoch": 0.33665499596014004,
+      "grad_norm": 0.440519243478775,
+      "learning_rate": 2e-05,
+      "loss": 0.8608,
+      "step": 3750
+    },
+    {
+      "epoch": 0.33755274261603374,
+      "grad_norm": 0.5013626217842102,
+      "learning_rate": 2e-05,
+      "loss": 0.8917,
+      "step": 3760
+    },
+    {
+      "epoch": 0.33845048927192745,
+      "grad_norm": 0.715194582939148,
+      "learning_rate": 2e-05,
+      "loss": 0.8734,
+      "step": 3770
+    },
+    {
+      "epoch": 0.33934823592782115,
+      "grad_norm": 0.47492650151252747,
+      "learning_rate": 2e-05,
+      "loss": 0.9135,
+      "step": 3780
+    },
+    {
+      "epoch": 0.34024598258371486,
+      "grad_norm": 0.4974531829357147,
+      "learning_rate": 2e-05,
+      "loss": 0.8346,
+      "step": 3790
+    },
+    {
+      "epoch": 0.34114372923960856,
+      "grad_norm": 0.44928160309791565,
+      "learning_rate": 2e-05,
+      "loss": 0.9073,
+      "step": 3800
+    },
+    {
+      "epoch": 0.34204147589550227,
+      "grad_norm": 0.45188620686531067,
+      "learning_rate": 2e-05,
+      "loss": 0.9242,
+      "step": 3810
+    },
+    {
+      "epoch": 0.342939222551396,
+      "grad_norm": 0.4332706928253174,
+      "learning_rate": 2e-05,
+      "loss": 0.8118,
+      "step": 3820
+    },
+    {
+      "epoch": 0.3438369692072897,
+      "grad_norm": 0.7933923602104187,
+      "learning_rate": 2e-05,
+      "loss": 0.8813,
+      "step": 3830
+    },
+    {
+      "epoch": 0.3447347158631834,
+      "grad_norm": 0.4981561601161957,
+      "learning_rate": 2e-05,
+      "loss": 0.8946,
+      "step": 3840
+    },
+    {
+      "epoch": 0.34563246251907714,
+      "grad_norm": 0.534538745880127,
+      "learning_rate": 2e-05,
+      "loss": 0.8729,
+      "step": 3850
+    },
+    {
+      "epoch": 0.34653020917497085,
+      "grad_norm": 0.48810383677482605,
+      "learning_rate": 2e-05,
+      "loss": 0.8424,
+      "step": 3860
+    },
+    {
+      "epoch": 0.34742795583086455,
+      "grad_norm": 0.46285027265548706,
+      "learning_rate": 2e-05,
+      "loss": 0.9506,
+      "step": 3870
+    },
+    {
+      "epoch": 0.34832570248675826,
+      "grad_norm": 0.4777646064758301,
+      "learning_rate": 2e-05,
+      "loss": 0.9019,
+      "step": 3880
+    },
+    {
+      "epoch": 0.34922344914265196,
+      "grad_norm": 0.489434152841568,
+      "learning_rate": 2e-05,
+      "loss": 0.9669,
+      "step": 3890
+    },
+    {
+      "epoch": 0.35012119579854567,
+      "grad_norm": 0.5317792296409607,
+      "learning_rate": 2e-05,
+      "loss": 0.7988,
+      "step": 3900
+    },
+    {
+      "epoch": 0.3510189424544394,
+      "grad_norm": 0.5085530877113342,
+      "learning_rate": 2e-05,
+      "loss": 0.9042,
+      "step": 3910
+    },
+    {
+      "epoch": 0.3519166891103331,
+      "grad_norm": 1.0325146913528442,
+      "learning_rate": 2e-05,
+      "loss": 0.9211,
+      "step": 3920
+    },
+    {
+      "epoch": 0.3528144357662268,
+      "grad_norm": 0.5225204825401306,
+      "learning_rate": 2e-05,
+      "loss": 0.8261,
+      "step": 3930
+    },
+    {
+      "epoch": 0.3537121824221205,
+      "grad_norm": 0.48252421617507935,
+      "learning_rate": 2e-05,
+      "loss": 0.8958,
+      "step": 3940
+    },
+    {
+      "epoch": 0.3546099290780142,
+      "grad_norm": 0.46308305859565735,
+      "learning_rate": 2e-05,
+      "loss": 0.9022,
+      "step": 3950
+    },
+    {
+      "epoch": 0.3555076757339079,
+      "grad_norm": 0.4817369878292084,
+      "learning_rate": 2e-05,
+      "loss": 0.8334,
+      "step": 3960
+    },
+    {
+      "epoch": 0.3564054223898016,
+      "grad_norm": 1.1378978490829468,
+      "learning_rate": 2e-05,
+      "loss": 0.9141,
+      "step": 3970
+    },
+    {
+      "epoch": 0.3573031690456953,
+      "grad_norm": 0.46689313650131226,
+      "learning_rate": 2e-05,
+      "loss": 0.9098,
+      "step": 3980
+    },
+    {
+      "epoch": 0.358200915701589,
+      "grad_norm": 0.528131365776062,
+      "learning_rate": 2e-05,
+      "loss": 0.9457,
+      "step": 3990
+    },
+    {
+      "epoch": 0.3590986623574827,
+      "grad_norm": 0.5105645656585693,
+      "learning_rate": 2e-05,
+      "loss": 0.907,
+      "step": 4000
+    },
+    {
+      "epoch": 0.35999640901337643,
+      "grad_norm": 0.4819955825805664,
+      "learning_rate": 2e-05,
+      "loss": 0.8658,
+      "step": 4010
+    },
+    {
+      "epoch": 0.36089415566927013,
+      "grad_norm": 0.4840661287307739,
+      "learning_rate": 2e-05,
+      "loss": 0.8925,
+      "step": 4020
+    },
+    {
+      "epoch": 0.36179190232516384,
+      "grad_norm": 0.5123524069786072,
+      "learning_rate": 2e-05,
+      "loss": 0.8747,
+      "step": 4030
+    },
+    {
+      "epoch": 0.36268964898105754,
+      "grad_norm": 0.4902634024620056,
+      "learning_rate": 2e-05,
+      "loss": 0.8479,
+      "step": 4040
+    },
+    {
+      "epoch": 0.36358739563695125,
+      "grad_norm": 0.5080075263977051,
+      "learning_rate": 2e-05,
+      "loss": 0.8582,
+      "step": 4050
+    },
+    {
+      "epoch": 0.36448514229284495,
+      "grad_norm": 0.47767189145088196,
+      "learning_rate": 2e-05,
+      "loss": 0.8954,
+      "step": 4060
+    },
+    {
+      "epoch": 0.36538288894873866,
+      "grad_norm": 0.5419741868972778,
+      "learning_rate": 2e-05,
+      "loss": 0.8841,
+      "step": 4070
+    },
+    {
+      "epoch": 0.36628063560463237,
+      "grad_norm": 0.5210027098655701,
+      "learning_rate": 2e-05,
+      "loss": 0.8652,
+      "step": 4080
+    },
+    {
+      "epoch": 0.36717838226052607,
+      "grad_norm": 0.49972549080848694,
+      "learning_rate": 2e-05,
+      "loss": 0.8438,
+      "step": 4090
+    },
+    {
+      "epoch": 0.3680761289164198,
+      "grad_norm": 0.4870189130306244,
+      "learning_rate": 2e-05,
+      "loss": 0.8632,
+      "step": 4100
+    },
+    {
+      "epoch": 0.3689738755723135,
+      "grad_norm": 0.49740269780158997,
+      "learning_rate": 2e-05,
+      "loss": 0.8674,
+      "step": 4110
+    },
+    {
+      "epoch": 0.3698716222282072,
+      "grad_norm": 0.4444533884525299,
+      "learning_rate": 2e-05,
+      "loss": 0.8351,
+      "step": 4120
+    },
+    {
+      "epoch": 0.3707693688841009,
+      "grad_norm": 0.45677250623703003,
+      "learning_rate": 2e-05,
+      "loss": 0.886,
+      "step": 4130
+    },
+    {
+      "epoch": 0.3716671155399946,
+      "grad_norm": 0.5022479295730591,
+      "learning_rate": 2e-05,
+      "loss": 0.8777,
+      "step": 4140
+    },
+    {
+      "epoch": 0.3725648621958883,
+      "grad_norm": 0.45390111207962036,
+      "learning_rate": 2e-05,
+      "loss": 0.8553,
+      "step": 4150
+    },
+    {
+      "epoch": 0.373462608851782,
+      "grad_norm": 0.4950340688228607,
+      "learning_rate": 2e-05,
+      "loss": 0.8713,
+      "step": 4160
+    },
+    {
+      "epoch": 0.3743603555076757,
+      "grad_norm": 0.4702810049057007,
+      "learning_rate": 2e-05,
+      "loss": 0.8994,
+      "step": 4170
+    },
+    {
+      "epoch": 0.3752581021635694,
+      "grad_norm": 0.45414838194847107,
+      "learning_rate": 2e-05,
+      "loss": 0.9283,
+      "step": 4180
+    },
+    {
+      "epoch": 0.3761558488194631,
+      "grad_norm": 0.518036961555481,
+      "learning_rate": 2e-05,
+      "loss": 0.9029,
+      "step": 4190
+    },
+    {
+      "epoch": 0.37705359547535683,
+      "grad_norm": 0.43749964237213135,
+      "learning_rate": 2e-05,
+      "loss": 0.8557,
+      "step": 4200
+    },
+    {
+      "epoch": 0.37795134213125053,
+      "grad_norm": 0.4743707776069641,
+      "learning_rate": 2e-05,
+      "loss": 0.8276,
+      "step": 4210
+    },
+    {
+      "epoch": 0.3788490887871443,
+      "grad_norm": 0.4764813482761383,
+      "learning_rate": 2e-05,
+      "loss": 0.8768,
+      "step": 4220
+    },
+    {
+      "epoch": 0.379746835443038,
+      "grad_norm": 0.4532718062400818,
+      "learning_rate": 2e-05,
+      "loss": 0.919,
+      "step": 4230
+    },
+    {
+      "epoch": 0.3806445820989317,
+      "grad_norm": 0.466553658246994,
+      "learning_rate": 2e-05,
+      "loss": 0.8526,
+      "step": 4240
+    },
+    {
+      "epoch": 0.3815423287548254,
+      "grad_norm": 0.5103240609169006,
+      "learning_rate": 2e-05,
+      "loss": 0.8529,
+      "step": 4250
+    },
+    {
+      "epoch": 0.3824400754107191,
+      "grad_norm": 0.4421781599521637,
+      "learning_rate": 2e-05,
+      "loss": 0.9116,
+      "step": 4260
+    },
+    {
+      "epoch": 0.3833378220666128,
+      "grad_norm": 0.47624221444129944,
+      "learning_rate": 2e-05,
+      "loss": 0.906,
+      "step": 4270
+    },
+    {
+      "epoch": 0.3842355687225065,
+      "grad_norm": 0.5574905872344971,
+      "learning_rate": 2e-05,
+      "loss": 0.9194,
+      "step": 4280
+    },
+    {
+      "epoch": 0.38513331537840023,
+      "grad_norm": 0.47859275341033936,
+      "learning_rate": 2e-05,
+      "loss": 0.8999,
+      "step": 4290
+    },
+    {
+      "epoch": 0.38603106203429394,
+      "grad_norm": 0.4639357030391693,
+      "learning_rate": 2e-05,
+      "loss": 0.8938,
+      "step": 4300
+    },
+    {
+      "epoch": 0.38692880869018764,
+      "grad_norm": 0.43172457814216614,
+      "learning_rate": 2e-05,
+      "loss": 0.8906,
+      "step": 4310
+    },
+    {
+      "epoch": 0.38782655534608135,
+      "grad_norm": 0.4365277886390686,
+      "learning_rate": 2e-05,
+      "loss": 0.8923,
+      "step": 4320
+    },
+    {
+      "epoch": 0.38872430200197505,
+      "grad_norm": 0.4546705186367035,
+      "learning_rate": 2e-05,
+      "loss": 0.9221,
+      "step": 4330
+    },
+    {
+      "epoch": 0.38962204865786876,
+      "grad_norm": 0.49238309264183044,
+      "learning_rate": 2e-05,
+      "loss": 0.8449,
+      "step": 4340
+    },
+    {
+      "epoch": 0.39051979531376246,
+      "grad_norm": 0.47965800762176514,
+      "learning_rate": 2e-05,
+      "loss": 0.8897,
+      "step": 4350
+    },
+    {
+      "epoch": 0.39141754196965617,
+      "grad_norm": 0.5566319823265076,
+      "learning_rate": 2e-05,
+      "loss": 0.8452,
+      "step": 4360
+    },
+    {
+      "epoch": 0.3923152886255499,
+      "grad_norm": 0.4641070067882538,
+      "learning_rate": 2e-05,
+      "loss": 0.8889,
+      "step": 4370
+    },
+    {
+      "epoch": 0.3932130352814436,
+      "grad_norm": 0.43325361609458923,
+      "learning_rate": 2e-05,
+      "loss": 0.8459,
+      "step": 4380
+    },
+    {
+      "epoch": 0.3941107819373373,
+      "grad_norm": 0.4486607313156128,
+      "learning_rate": 2e-05,
+      "loss": 0.8451,
+      "step": 4390
+    },
+    {
+      "epoch": 0.395008528593231,
+      "grad_norm": 0.44522610306739807,
+      "learning_rate": 2e-05,
+      "loss": 0.8923,
+      "step": 4400
+    },
+    {
+      "epoch": 0.3959062752491247,
+      "grad_norm": 0.4850371181964874,
+      "learning_rate": 2e-05,
+      "loss": 0.868,
+      "step": 4410
+    },
+    {
+      "epoch": 0.3968040219050184,
+      "grad_norm": 0.4501579701900482,
+      "learning_rate": 2e-05,
+      "loss": 0.8864,
+      "step": 4420
+    },
+    {
+      "epoch": 0.3977017685609121,
+      "grad_norm": 0.43735620379447937,
+      "learning_rate": 2e-05,
+      "loss": 0.8297,
+      "step": 4430
+    },
+    {
+      "epoch": 0.3985995152168058,
+      "grad_norm": 0.4607102870941162,
+      "learning_rate": 2e-05,
+      "loss": 0.8621,
+      "step": 4440
+    },
+    {
+      "epoch": 0.3994972618726995,
+      "grad_norm": 0.4893702566623688,
+      "learning_rate": 2e-05,
+      "loss": 0.9005,
+      "step": 4450
+    },
+    {
+      "epoch": 0.4003950085285932,
+      "grad_norm": 0.4818199574947357,
+      "learning_rate": 2e-05,
+      "loss": 0.8503,
+      "step": 4460
+    },
+    {
+      "epoch": 0.4012927551844869,
+      "grad_norm": 0.4866369664669037,
+      "learning_rate": 2e-05,
+      "loss": 0.8723,
+      "step": 4470
+    },
+    {
+      "epoch": 0.40219050184038063,
+      "grad_norm": 0.7129011750221252,
+      "learning_rate": 2e-05,
+      "loss": 0.9036,
+      "step": 4480
+    },
+    {
+      "epoch": 0.40308824849627434,
+      "grad_norm": 0.48549818992614746,
+      "learning_rate": 2e-05,
+      "loss": 0.9047,
+      "step": 4490
+    },
+    {
+      "epoch": 0.40398599515216804,
+      "grad_norm": 0.6809606552124023,
+      "learning_rate": 2e-05,
+      "loss": 0.8368,
+      "step": 4500
+    },
+    {
+      "epoch": 0.40488374180806175,
+      "grad_norm": 0.4583209455013275,
+      "learning_rate": 2e-05,
+      "loss": 0.8887,
+      "step": 4510
+    },
+    {
+      "epoch": 0.40578148846395545,
+      "grad_norm": 0.47108474373817444,
+      "learning_rate": 2e-05,
+      "loss": 0.8338,
+      "step": 4520
+    },
+    {
+      "epoch": 0.40667923511984916,
+      "grad_norm": 0.4781590700149536,
+      "learning_rate": 2e-05,
+      "loss": 0.9011,
+      "step": 4530
+    },
+    {
+      "epoch": 0.40757698177574286,
+      "grad_norm": 0.5096138715744019,
+      "learning_rate": 2e-05,
+      "loss": 0.9116,
+      "step": 4540
+    },
+    {
+      "epoch": 0.40847472843163657,
+      "grad_norm": 0.4841920733451843,
+      "learning_rate": 2e-05,
+      "loss": 0.8933,
+      "step": 4550
+    },
+    {
+      "epoch": 0.4093724750875303,
+      "grad_norm": 0.46508246660232544,
+      "learning_rate": 2e-05,
+      "loss": 0.9177,
+      "step": 4560
+    },
+    {
+      "epoch": 0.410270221743424,
+      "grad_norm": 0.5557758212089539,
+      "learning_rate": 2e-05,
+      "loss": 0.8685,
+      "step": 4570
+    },
+    {
+      "epoch": 0.4111679683993177,
+      "grad_norm": 0.44160884618759155,
+      "learning_rate": 2e-05,
+      "loss": 0.8669,
+      "step": 4580
+    },
+    {
+      "epoch": 0.41206571505521145,
+      "grad_norm": 0.46800750494003296,
+      "learning_rate": 2e-05,
+      "loss": 0.8677,
+      "step": 4590
+    },
+    {
+      "epoch": 0.41296346171110515,
+      "grad_norm": 0.45948630571365356,
+      "learning_rate": 2e-05,
+      "loss": 0.8303,
+      "step": 4600
+    },
+    {
+      "epoch": 0.41386120836699886,
+      "grad_norm": 0.4520060420036316,
+      "learning_rate": 2e-05,
+      "loss": 0.8189,
+      "step": 4610
+    },
+    {
+      "epoch": 0.41475895502289256,
+      "grad_norm": 0.5067851543426514,
+      "learning_rate": 2e-05,
+      "loss": 0.8771,
+      "step": 4620
+    },
+    {
+      "epoch": 0.41565670167878627,
+      "grad_norm": 0.6326944828033447,
+      "learning_rate": 2e-05,
+      "loss": 0.8963,
+      "step": 4630
+    },
+    {
+      "epoch": 0.41655444833467997,
+      "grad_norm": 0.4706971347332001,
+      "learning_rate": 2e-05,
+      "loss": 0.8849,
+      "step": 4640
+    },
+    {
+      "epoch": 0.4174521949905737,
+      "grad_norm": 0.5517380833625793,
+      "learning_rate": 2e-05,
+      "loss": 0.8851,
+      "step": 4650
+    },
+    {
+      "epoch": 0.4183499416464674,
+      "grad_norm": 0.47117501497268677,
+      "learning_rate": 2e-05,
+      "loss": 0.7964,
+      "step": 4660
+    },
+    {
+      "epoch": 0.4192476883023611,
+      "grad_norm": 0.45700523257255554,
+      "learning_rate": 2e-05,
+      "loss": 0.8469,
+      "step": 4670
+    },
+    {
+      "epoch": 0.4201454349582548,
+      "grad_norm": 0.4756954312324524,
+      "learning_rate": 2e-05,
+      "loss": 0.8898,
+      "step": 4680
+    },
+    {
+      "epoch": 0.4210431816141485,
+      "grad_norm": 0.46773287653923035,
+      "learning_rate": 2e-05,
+      "loss": 0.9193,
+      "step": 4690
+    },
+    {
+      "epoch": 0.4219409282700422,
+      "grad_norm": 0.44644755125045776,
+      "learning_rate": 2e-05,
+      "loss": 0.845,
+      "step": 4700
+    },
+    {
+      "epoch": 0.4228386749259359,
+      "grad_norm": 0.5897283554077148,
+      "learning_rate": 2e-05,
+      "loss": 0.897,
+      "step": 4710
+    },
+    {
+      "epoch": 0.4237364215818296,
+      "grad_norm": 0.44933563470840454,
+      "learning_rate": 2e-05,
+      "loss": 0.8448,
+      "step": 4720
+    },
+    {
+      "epoch": 0.4246341682377233,
+      "grad_norm": 0.5093523263931274,
+      "learning_rate": 2e-05,
+      "loss": 0.8938,
+      "step": 4730
+    },
+    {
+      "epoch": 0.425531914893617,
+      "grad_norm": 0.4830954372882843,
+      "learning_rate": 2e-05,
+      "loss": 0.8331,
+      "step": 4740
+    },
+    {
+      "epoch": 0.42642966154951073,
+      "grad_norm": 0.4522993266582489,
+      "learning_rate": 2e-05,
+      "loss": 0.8971,
+      "step": 4750
+    },
+    {
+      "epoch": 0.42732740820540444,
+      "grad_norm": 0.46119436621665955,
+      "learning_rate": 2e-05,
+      "loss": 0.891,
+      "step": 4760
+    },
+    {
+      "epoch": 0.42822515486129814,
+      "grad_norm": 0.46443262696266174,
+      "learning_rate": 2e-05,
+      "loss": 0.8438,
+      "step": 4770
+    },
+    {
+      "epoch": 0.42912290151719185,
+      "grad_norm": 0.4841867983341217,
+      "learning_rate": 2e-05,
+      "loss": 0.8641,
+      "step": 4780
+    },
+    {
+      "epoch": 0.43002064817308555,
+      "grad_norm": 0.4359630048274994,
+      "learning_rate": 2e-05,
+      "loss": 0.808,
+      "step": 4790
+    },
+    {
+      "epoch": 0.43091839482897926,
+      "grad_norm": 0.5121234059333801,
+      "learning_rate": 2e-05,
+      "loss": 0.8834,
+      "step": 4800
+    },
+    {
+      "epoch": 0.43181614148487296,
+      "grad_norm": 0.5362908840179443,
+      "learning_rate": 2e-05,
+      "loss": 0.8869,
+      "step": 4810
+    },
+    {
+      "epoch": 0.43271388814076667,
+      "grad_norm": 0.47759607434272766,
+      "learning_rate": 2e-05,
+      "loss": 0.9063,
+      "step": 4820
+    },
+    {
+      "epoch": 0.4336116347966604,
+      "grad_norm": 0.49203813076019287,
+      "learning_rate": 2e-05,
+      "loss": 0.9002,
+      "step": 4830
+    },
+    {
+      "epoch": 0.4345093814525541,
+      "grad_norm": 0.5474998950958252,
+      "learning_rate": 2e-05,
+      "loss": 0.9314,
+      "step": 4840
+    },
+    {
+      "epoch": 0.4354071281084478,
+      "grad_norm": 0.45726820826530457,
+      "learning_rate": 2e-05,
+      "loss": 0.8687,
+      "step": 4850
+    },
+    {
+      "epoch": 0.4363048747643415,
+      "grad_norm": 0.47808167338371277,
+      "learning_rate": 2e-05,
+      "loss": 0.8866,
+      "step": 4860
+    },
+    {
+      "epoch": 0.4372026214202352,
+      "grad_norm": 0.606437623500824,
+      "learning_rate": 2e-05,
+      "loss": 0.8403,
+      "step": 4870
+    },
+    {
+      "epoch": 0.4381003680761289,
+      "grad_norm": 0.559087872505188,
+      "learning_rate": 2e-05,
+      "loss": 0.8418,
+      "step": 4880
+    },
+    {
+      "epoch": 0.4389981147320226,
+      "grad_norm": 0.49517837166786194,
+      "learning_rate": 2e-05,
+      "loss": 0.9058,
+      "step": 4890
+    },
+    {
+      "epoch": 0.4398958613879163,
+      "grad_norm": 0.7305907607078552,
+      "learning_rate": 2e-05,
+      "loss": 0.8721,
+      "step": 4900
+    },
+    {
+      "epoch": 0.44079360804381,
+      "grad_norm": 0.45854124426841736,
+      "learning_rate": 2e-05,
+      "loss": 0.8791,
+      "step": 4910
+    },
+    {
+      "epoch": 0.4416913546997037,
+      "grad_norm": 0.46924304962158203,
+      "learning_rate": 2e-05,
+      "loss": 0.8213,
+      "step": 4920
+    },
+    {
+      "epoch": 0.4425891013555974,
+      "grad_norm": 0.45977941155433655,
+      "learning_rate": 2e-05,
+      "loss": 0.8435,
+      "step": 4930
+    },
+    {
+      "epoch": 0.44348684801149113,
+      "grad_norm": 0.552426278591156,
+      "learning_rate": 2e-05,
+      "loss": 0.8749,
+      "step": 4940
+    },
+    {
+      "epoch": 0.4443845946673849,
+      "grad_norm": 0.7902478575706482,
+      "learning_rate": 2e-05,
+      "loss": 0.8477,
+      "step": 4950
+    },
+    {
+      "epoch": 0.4452823413232786,
+      "grad_norm": 0.5436770915985107,
+      "learning_rate": 2e-05,
+      "loss": 0.8834,
+      "step": 4960
+    },
+    {
+      "epoch": 0.4461800879791723,
+      "grad_norm": 0.4446653425693512,
+      "learning_rate": 2e-05,
+      "loss": 0.7737,
+      "step": 4970
+    },
+    {
+      "epoch": 0.447077834635066,
+      "grad_norm": 0.5170835256576538,
+      "learning_rate": 2e-05,
+      "loss": 0.8161,
+      "step": 4980
+    },
+    {
+      "epoch": 0.4479755812909597,
+      "grad_norm": 0.5019964575767517,
+      "learning_rate": 2e-05,
+      "loss": 0.8499,
+      "step": 4990
+    },
+    {
+      "epoch": 0.4488733279468534,
+      "grad_norm": 0.4614900052547455,
+      "learning_rate": 2e-05,
+      "loss": 0.8798,
+      "step": 5000
+    },
+    {
+      "epoch": 0.4497710746027471,
+      "grad_norm": 0.4722237288951874,
+      "learning_rate": 2e-05,
+      "loss": 0.9688,
+      "step": 5010
+    },
+    {
+      "epoch": 0.45066882125864083,
+      "grad_norm": 0.5104867219924927,
+      "learning_rate": 2e-05,
+      "loss": 0.9173,
+      "step": 5020
+    },
+    {
+      "epoch": 0.45156656791453453,
+      "grad_norm": 0.4620506763458252,
+      "learning_rate": 2e-05,
+      "loss": 0.8627,
+      "step": 5030
+    },
+    {
+      "epoch": 0.45246431457042824,
+      "grad_norm": 0.49297621846199036,
+      "learning_rate": 2e-05,
+      "loss": 0.9026,
+      "step": 5040
+    },
+    {
+      "epoch": 0.45336206122632194,
+      "grad_norm": 0.492024302482605,
+      "learning_rate": 2e-05,
+      "loss": 0.8482,
+      "step": 5050
+    },
+    {
+      "epoch": 0.45425980788221565,
+      "grad_norm": 0.4808714687824249,
+      "learning_rate": 2e-05,
+      "loss": 0.8596,
+      "step": 5060
+    },
+    {
+      "epoch": 0.45515755453810935,
+      "grad_norm": 0.488479882478714,
+      "learning_rate": 2e-05,
+      "loss": 0.8524,
+      "step": 5070
+    },
+    {
+      "epoch": 0.45605530119400306,
+      "grad_norm": 0.4880830645561218,
+      "learning_rate": 2e-05,
+      "loss": 0.8161,
+      "step": 5080
+    },
+    {
+      "epoch": 0.45695304784989677,
+      "grad_norm": 0.47819364070892334,
+      "learning_rate": 1.9999810479633703e-05,
+      "loss": 0.7741,
+      "step": 5090
+    },
+    {
+      "epoch": 0.45785079450579047,
+      "grad_norm": 0.5089237689971924,
+      "learning_rate": 1.997707677020786e-05,
+      "loss": 0.8417,
+      "step": 5100
+    },
+    {
+      "epoch": 0.4587485411616842,
+      "grad_norm": 0.750914454460144,
+      "learning_rate": 1.9916538196124637e-05,
+      "loss": 0.8694,
+      "step": 5110
+    },
+    {
+      "epoch": 0.4596462878175779,
+      "grad_norm": 0.5125513076782227,
+      "learning_rate": 1.9818425303863402e-05,
+      "loss": 0.8408,
+      "step": 5120
+    },
+    {
+      "epoch": 0.4605440344734716,
+      "grad_norm": 0.5394951105117798,
+      "learning_rate": 1.9683111732582887e-05,
+      "loss": 0.8755,
+      "step": 5130
+    },
+    {
+      "epoch": 0.4614417811293653,
+      "grad_norm": 0.4500407874584198,
+      "learning_rate": 1.951111279120706e-05,
+      "loss": 0.7917,
+      "step": 5140
+    },
+    {
+      "epoch": 0.462339527785259,
+      "grad_norm": 0.48868054151535034,
+      "learning_rate": 1.9303083495996028e-05,
+      "loss": 0.7847,
+      "step": 5150
+    },
+    {
+      "epoch": 0.4632372744411527,
+      "grad_norm": 0.4962034225463867,
+      "learning_rate": 1.9059816076075482e-05,
+      "loss": 0.8769,
+      "step": 5160
+    },
+    {
+      "epoch": 0.4641350210970464,
+      "grad_norm": 0.4250297248363495,
+      "learning_rate": 1.8782236956424142e-05,
+      "loss": 0.7992,
+      "step": 5170
+    },
+    {
+      "epoch": 0.4650327677529401,
+      "grad_norm": 0.5036498308181763,
+      "learning_rate": 1.8471403229808897e-05,
+      "loss": 0.8254,
+      "step": 5180
+    },
+    {
+      "epoch": 0.4659305144088338,
+      "grad_norm": 0.5076369643211365,
+      "learning_rate": 1.8128498631103333e-05,
+      "loss": 0.8724,
+      "step": 5190
+    },
+    {
+      "epoch": 0.4668282610647275,
+      "grad_norm": 0.4649927318096161,
+      "learning_rate": 1.7754829029320507e-05,
+      "loss": 0.8527,
+      "step": 5200
+    },
+    {
+      "epoch": 0.46772600772062123,
+      "grad_norm": 0.403854101896286,
+      "learning_rate": 1.7351817454527473e-05,
+      "loss": 0.7918,
+      "step": 5210
+    },
+    {
+      "epoch": 0.46862375437651493,
+      "grad_norm": 0.4863133430480957,
+      "learning_rate": 1.6920998678580345e-05,
+      "loss": 0.8084,
+      "step": 5220
+    },
+    {
+      "epoch": 0.46952150103240864,
+      "grad_norm": 0.5051937103271484,
+      "learning_rate": 1.6464013370317854e-05,
+      "loss": 0.836,
+      "step": 5230
+    },
+    {
+      "epoch": 0.47041924768830234,
+      "grad_norm": 0.5071216225624084,
+      "learning_rate": 1.5982601847471915e-05,
+      "loss": 0.841,
+      "step": 5240
+    },
+    {
+      "epoch": 0.47131699434419605,
+      "grad_norm": 0.5376131534576416,
+      "learning_rate": 1.5478597449089553e-05,
+      "loss": 0.8971,
+      "step": 5250
+    },
+    {
+      "epoch": 0.47221474100008975,
+      "grad_norm": 0.4799082279205322,
+      "learning_rate": 1.4953919553705725e-05,
+      "loss": 0.8462,
+      "step": 5260
+    },
+    {
+      "epoch": 0.47311248765598346,
+      "grad_norm": 0.44629716873168945,
+      "learning_rate": 1.441056626985559e-05,
+      "loss": 0.8469,
+      "step": 5270
+    },
+    {
+      "epoch": 0.47401023431187717,
+      "grad_norm": 0.5925596952438354,
+      "learning_rate": 1.3850606826762668e-05,
+      "loss": 0.799,
+      "step": 5280
+    },
+    {
+      "epoch": 0.47490798096777087,
+      "grad_norm": 0.46540284156799316,
+      "learning_rate": 1.3276173694181128e-05,
+      "loss": 0.8349,
+      "step": 5290
+    },
+    {
+      "epoch": 0.4758057276236646,
+      "grad_norm": 0.4988724887371063,
+      "learning_rate": 1.2689454461401822e-05,
+      "loss": 0.7874,
+      "step": 5300
+    },
+    {
+      "epoch": 0.4767034742795583,
+      "grad_norm": 0.48345696926116943,
+      "learning_rate": 1.2092683506348939e-05,
+      "loss": 0.7854,
+      "step": 5310
+    },
+    {
+      "epoch": 0.47760122093545204,
+      "grad_norm": 0.4852100610733032,
+      "learning_rate": 1.14881334864935e-05,
+      "loss": 0.8265,
+      "step": 5320
+    },
+    {
+      "epoch": 0.47849896759134575,
+      "grad_norm": 0.4882681667804718,
+      "learning_rate": 1.0878106683988481e-05,
+      "loss": 0.8637,
+      "step": 5330
+    },
+    {
+      "epoch": 0.47939671424723945,
+      "grad_norm": 0.45849791169166565,
+      "learning_rate": 1.026492623798551e-05,
+      "loss": 0.7507,
+      "step": 5340
+    },
+    {
+      "epoch": 0.48029446090313316,
+      "grad_norm": 0.47716638445854187,
+      "learning_rate": 9.650927297522743e-06,
+      "loss": 0.8437,
+      "step": 5350
+    },
+    {
+      "epoch": 0.48119220755902686,
+      "grad_norm": 0.4413863718509674,
+      "learning_rate": 9.038448128676012e-06,
+      "loss": 0.8547,
+      "step": 5360
+    },
+    {
+      "epoch": 0.48208995421492057,
+      "grad_norm": 0.5485756993293762,
+      "learning_rate": 8.42982120983949e-06,
+      "loss": 0.9039,
+      "step": 5370
+    },
+    {
+      "epoch": 0.4829877008708143,
+      "grad_norm": 0.4490965008735657,
+      "learning_rate": 7.827364349047303e-06,
+      "loss": 0.7911,
+      "step": 5380
+    },
+    {
+      "epoch": 0.483885447526708,
+      "grad_norm": 0.5348581671714783,
+      "learning_rate": 7.233371857163706e-06,
+      "loss": 0.804,
+      "step": 5390
+    },
+    {
+      "epoch": 0.4847831941826017,
+      "grad_norm": 0.45429131388664246,
+      "learning_rate": 6.650105810556492e-06,
+      "loss": 0.7966,
+      "step": 5400
+    },
+    {
+      "epoch": 0.4856809408384954,
+      "grad_norm": 0.472555547952652,
+      "learning_rate": 6.079787436527681e-06,
+      "loss": 0.7768,
+      "step": 5410
+    },
+    {
+      "epoch": 0.4865786874943891,
+      "grad_norm": 0.44279059767723083,
+      "learning_rate": 5.524588654307961e-06,
+      "loss": 0.8754,
+      "step": 5420
+    },
+    {
+      "epoch": 0.4874764341502828,
+      "grad_norm": 0.45473113656044006,
+      "learning_rate": 4.986623803828985e-06,
+      "loss": 0.8486,
+      "step": 5430
+    },
+    {
+      "epoch": 0.4883741808061765,
+      "grad_norm": 0.42961612343788147,
+      "learning_rate": 4.467941593772423e-06,
+      "loss": 0.8041,
+      "step": 5440
+    },
+    {
+      "epoch": 0.4892719274620702,
+      "grad_norm": 0.4712171256542206,
+      "learning_rate": 3.9705172995597606e-06,
+      "loss": 0.8244,
+      "step": 5450
+    },
+    {
+      "epoch": 0.4901696741179639,
+      "grad_norm": 0.4455659091472626,
+      "learning_rate": 3.4962452409948346e-06,
+      "loss": 0.8175,
+      "step": 5460
+    },
+    {
+      "epoch": 0.4910674207738576,
+      "grad_norm": 0.41610950231552124,
+      "learning_rate": 3.046931568206211e-06,
+      "loss": 0.7875,
+      "step": 5470
+    },
+    {
+      "epoch": 0.4919651674297513,
+      "grad_norm": 0.4421256482601166,
+      "learning_rate": 2.624287383362404e-06,
+      "loss": 0.8289,
+      "step": 5480
+    },
+    {
+      "epoch": 0.49286291408564503,
+      "grad_norm": 0.5148488283157349,
+      "learning_rate": 2.2299222243541845e-06,
+      "loss": 0.8606,
+      "step": 5490
+    },
+    {
+      "epoch": 0.49376066074153874,
+      "grad_norm": 0.45190247893333435,
+      "learning_rate": 1.8653379352598246e-06,
+      "loss": 0.7984,
+      "step": 5500
+    },
+    {
+      "epoch": 0.49465840739743244,
+      "grad_norm": 0.4787582755088806,
+      "learning_rate": 1.5319229469361075e-06,
+      "loss": 0.881,
+      "step": 5510
+    },
+    {
+      "epoch": 0.49555615405332615,
+      "grad_norm": 0.4484575092792511,
+      "learning_rate": 1.2309469895160917e-06,
+      "loss": 0.8711,
+      "step": 5520
+    },
+    {
+      "epoch": 0.49645390070921985,
+      "grad_norm": 0.5196358561515808,
+      "learning_rate": 9.635562569497974e-07,
+      "loss": 0.8399,
+      "step": 5530
+    },
+    {
+      "epoch": 0.49735164736511356,
+      "grad_norm": 0.41991645097732544,
+      "learning_rate": 7.307690420024867e-07,
+      "loss": 0.8138,
+      "step": 5540
+    },
+    {
+      "epoch": 0.49824939402100726,
+      "grad_norm": 0.45500171184539795,
+      "learning_rate": 5.334718583335942e-07,
+      "loss": 0.8633,
+      "step": 5550
+    },
+    {
+      "epoch": 0.49914714067690097,
+      "grad_norm": 0.5754929184913635,
+      "learning_rate": 3.7241606442442763e-07,
+      "loss": 0.8252,
+      "step": 5560
+    },
+    {
+      "epoch": 0.5000448873327947,
+      "grad_norm": 0.44744038581848145,
+      "learning_rate": 2.482150022115879e-07,
+      "loss": 0.8292,
+      "step": 5570
+    },
+    {
+      "epoch": 0.5009426339886884,
+      "grad_norm": 0.5219799876213074,
+      "learning_rate": 1.6134166132293174e-07,
+      "loss": 0.819,
+      "step": 5580
+    },
+    {
+      "epoch": 0.5018403806445821,
+      "grad_norm": 0.5051019787788391,
+      "learning_rate": 1.1212687781127357e-07,
+      "loss": 0.7834,
+      "step": 5590
+    },
+    {
+      "epoch": 0.5024688033037077,
+      "step": 5597,
+      "total_flos": 2.2737574718397743e+19,
+      "train_loss": 0.028161267840651417,
+      "train_runtime": 1789.279,
+      "train_samples_per_second": 37.537,
+      "train_steps_per_second": 3.128
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 5597,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 159,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2737574718397743e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

g2_1_oldcode_cm_newcode__edit/1000M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9346246973365617,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.003304087116779207,
+    "train_runtime": 610.4281,
+    "train_samples_per_second": 220.055,
+    "train_steps_per_second": 18.338
+}

g2_1_oldcode_cm_newcode__edit/1000M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g2_1_oldcode_cm_newcode__edit/1000M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g2_1_oldcode_cm_newcode__edit/1000M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g2_1_oldcode_cm_newcode__edit/1000M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g2_1_oldcode_cm_newcode__edit/1000M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

g2_1_oldcode_cm_newcode__edit/1000M/tokenizer_config.json

@@ -0,0 +1,191 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

g2_1_oldcode_cm_newcode__edit/1000M/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9346246973365617,
+    "total_flos": 4.547514943786923e+19,
+    "train_loss": 0.003304087116779207,
+    "train_runtime": 610.4281,
+    "train_samples_per_second": 220.055,
+    "train_steps_per_second": 18.338
+}

g2_1_oldcode_cm_newcode__edit/1000M/trainer_log.jsonl

@@ -0,0 +1,110 @@
+{"current_steps": 10180, "total_steps": 11194, "loss": 0.6082, "lr": 1.999957358086795e-05, "epoch": 0.8499624279869751, "percentage": 90.94, "elapsed_time": "0:00:37", "remaining_time": "0:00:03"}
+{"current_steps": 10190, "total_steps": 11194, "loss": 0.6326, "lr": 1.9991993813020483e-05, "epoch": 0.8507973616097521, "percentage": 91.03, "elapsed_time": "0:02:06", "remaining_time": "0:00:12"}
+{"current_steps": 10200, "total_steps": 11194, "loss": 0.5971, "lr": 1.9974946373028628e-05, "epoch": 0.851632295232529, "percentage": 91.12, "elapsed_time": "0:03:34", "remaining_time": "0:00:20"}
+{"current_steps": 10210, "total_steps": 11194, "loss": 0.5636, "lr": 1.994844749501764e-05, "epoch": 0.852467228855306, "percentage": 91.21, "elapsed_time": "0:05:03", "remaining_time": "0:00:29"}
+{"current_steps": 10220, "total_steps": 11194, "loss": 0.6031, "lr": 1.9912522413632988e-05, "epoch": 0.853302162478083, "percentage": 91.3, "elapsed_time": "0:06:31", "remaining_time": "0:00:37"}
+{"current_steps": 10230, "total_steps": 11194, "loss": 0.5787, "lr": 1.9867205340009656e-05, "epoch": 0.85413709610086, "percentage": 91.39, "elapsed_time": "0:08:00", "remaining_time": "0:00:45"}
+{"current_steps": 10240, "total_steps": 11194, "loss": 0.6111, "lr": 1.9812539429193164e-05, "epoch": 0.854972029723637, "percentage": 91.48, "elapsed_time": "0:09:28", "remaining_time": "0:00:52"}
+{"current_steps": 10250, "total_steps": 11194, "loss": 0.6276, "lr": 1.9748576739043423e-05, "epoch": 0.8558069633464139, "percentage": 91.57, "elapsed_time": "0:10:57", "remaining_time": "0:01:00"}
+{"current_steps": 10260, "total_steps": 11194, "loss": 0.5668, "lr": 1.967537818066047e-05, "epoch": 0.8566418969691909, "percentage": 91.66, "elapsed_time": "0:12:25", "remaining_time": "0:01:07"}
+{"current_steps": 10270, "total_steps": 11194, "loss": 0.5951, "lr": 1.959301346037941e-05, "epoch": 0.8574768305919679, "percentage": 91.75, "elapsed_time": "0:13:53", "remaining_time": "0:01:15"}
+{"current_steps": 10280, "total_steps": 11194, "loss": 0.6174, "lr": 1.9501561013389646e-05, "epoch": 0.858311764214745, "percentage": 91.83, "elapsed_time": "0:15:22", "remaining_time": "0:01:22"}
+{"current_steps": 10290, "total_steps": 11194, "loss": 0.5796, "lr": 1.9401107929041795e-05, "epoch": 0.859146697837522, "percentage": 91.92, "elapsed_time": "0:16:50", "remaining_time": "0:01:28"}
+{"current_steps": 10300, "total_steps": 11194, "loss": 0.5762, "lr": 1.9291749867913225e-05, "epoch": 0.8599816314602989, "percentage": 92.01, "elapsed_time": "0:18:19", "remaining_time": "0:01:35"}
+{"current_steps": 10310, "total_steps": 11194, "loss": 0.5754, "lr": 1.91735909707114e-05, "epoch": 0.8608165650830759, "percentage": 92.1, "elapsed_time": "0:19:47", "remaining_time": "0:01:41"}
+{"current_steps": 10320, "total_steps": 11194, "loss": 0.6103, "lr": 1.9046743759101545e-05, "epoch": 0.8616514987058529, "percentage": 92.19, "elapsed_time": "0:21:15", "remaining_time": "0:01:48"}
+{"current_steps": 10330, "total_steps": 11194, "loss": 0.6066, "lr": 1.8911329028553324e-05, "epoch": 0.8624864323286299, "percentage": 92.28, "elapsed_time": "0:22:44", "remaining_time": "0:01:54"}
+{"current_steps": 10340, "total_steps": 11194, "loss": 0.6241, "lr": 1.876747573330835e-05, "epoch": 0.8633213659514068, "percentage": 92.37, "elapsed_time": "0:24:50", "remaining_time": "0:02:03"}
+{"current_steps": 10350, "total_steps": 11194, "loss": 0.5879, "lr": 1.8615320863578244e-05, "epoch": 0.8641562995741838, "percentage": 92.46, "elapsed_time": "0:26:18", "remaining_time": "0:02:08"}
+{"current_steps": 10360, "total_steps": 11194, "loss": 0.6041, "lr": 1.8455009315090036e-05, "epoch": 0.8649912331969608, "percentage": 92.55, "elapsed_time": "0:27:46", "remaining_time": "0:02:14"}
+{"current_steps": 10370, "total_steps": 11194, "loss": 0.6092, "lr": 1.8286693751103294e-05, "epoch": 0.8658261668197378, "percentage": 92.64, "elapsed_time": "0:29:15", "remaining_time": "0:02:19"}
+{"current_steps": 10380, "total_steps": 11194, "loss": 0.6002, "lr": 1.811053445703024e-05, "epoch": 0.8666611004425148, "percentage": 92.73, "elapsed_time": "0:30:43", "remaining_time": "0:02:24"}
+{"current_steps": 10390, "total_steps": 11194, "loss": 0.6034, "lr": 1.7926699187797375e-05, "epoch": 0.8674960340652919, "percentage": 92.82, "elapsed_time": "0:32:12", "remaining_time": "0:02:29"}
+{"current_steps": 10400, "total_steps": 11194, "loss": 0.5968, "lr": 1.7735363008093995e-05, "epoch": 0.8683309676880688, "percentage": 92.91, "elapsed_time": "0:33:40", "remaining_time": "0:02:34"}
+{"current_steps": 10410, "total_steps": 11194, "loss": 0.6188, "lr": 1.7536708125659577e-05, "epoch": 0.8691659013108458, "percentage": 93.0, "elapsed_time": "0:35:09", "remaining_time": "0:02:38"}
+{"current_steps": 10420, "total_steps": 11194, "loss": 0.603, "lr": 1.733092371776905e-05, "epoch": 0.8700008349336228, "percentage": 93.09, "elapsed_time": "0:36:37", "remaining_time": "0:02:43"}
+{"current_steps": 10430, "total_steps": 11194, "loss": 0.5948, "lr": 1.7118205751080915e-05, "epoch": 0.8708357685563998, "percentage": 93.17, "elapsed_time": "0:38:05", "remaining_time": "0:02:47"}
+{"current_steps": 10440, "total_steps": 11194, "loss": 0.5895, "lr": 1.6898756795019935e-05, "epoch": 0.8716707021791767, "percentage": 93.26, "elapsed_time": "0:39:34", "remaining_time": "0:02:51"}
+{"current_steps": 10450, "total_steps": 11194, "loss": 0.6011, "lr": 1.667278582887211e-05, "epoch": 0.8725056358019537, "percentage": 93.35, "elapsed_time": "0:41:02", "remaining_time": "0:02:55"}
+{"current_steps": 10460, "total_steps": 11194, "loss": 0.6101, "lr": 1.6440508042775545e-05, "epoch": 0.8733405694247307, "percentage": 93.44, "elapsed_time": "0:42:31", "remaining_time": "0:02:59"}
+{"current_steps": 10470, "total_steps": 11194, "loss": 0.5977, "lr": 1.6202144632796796e-05, "epoch": 0.8741755030475077, "percentage": 93.53, "elapsed_time": "0:43:59", "remaining_time": "0:03:02"}
+{"current_steps": 10480, "total_steps": 11194, "loss": 0.5964, "lr": 1.595792259028783e-05, "epoch": 0.8750104366702847, "percentage": 93.62, "elapsed_time": "0:45:27", "remaining_time": "0:03:05"}
+{"current_steps": 10490, "total_steps": 11194, "loss": 0.5886, "lr": 1.570807448572419e-05, "epoch": 0.8758453702930618, "percentage": 93.71, "elapsed_time": "0:46:56", "remaining_time": "0:03:09"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.6136, "lr": 1.5452838247230197e-05, "epoch": 0.8766803039158387, "percentage": 93.8, "elapsed_time": "0:49:02", "remaining_time": "0:03:14"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.6032, "lr": 1.5192456934002106e-05, "epoch": 0.8775152375386157, "percentage": 93.89, "elapsed_time": "0:50:30", "remaining_time": "0:03:17"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.5786, "lr": 1.492717850484502e-05, "epoch": 0.8783501711613927, "percentage": 93.98, "elapsed_time": "0:51:59", "remaining_time": "0:03:19"}
+{"current_steps": 10500, "total_steps": 11194, "loss": 0.6097, "lr": 1.5452838247230197e-05, "epoch": 0.8766803039158387, "percentage": 93.8, "elapsed_time": "0:00:56", "remaining_time": "0:00:03"}
+{"current_steps": 10510, "total_steps": 11194, "loss": 0.6032, "lr": 1.5192456934002106e-05, "epoch": 0.8775152375386157, "percentage": 93.89, "elapsed_time": "0:02:25", "remaining_time": "0:00:09"}
+{"current_steps": 10520, "total_steps": 11194, "loss": 0.5786, "lr": 1.492717850484502e-05, "epoch": 0.8783501711613927, "percentage": 93.98, "elapsed_time": "0:03:54", "remaining_time": "0:00:15"}
+{"current_steps": 10530, "total_steps": 11194, "loss": 0.5685, "lr": 1.4657255582043904e-05, "epoch": 0.8791851047841697, "percentage": 94.07, "elapsed_time": "0:05:24", "remaining_time": "0:00:20"}
+{"current_steps": 10540, "total_steps": 11194, "loss": 0.5941, "lr": 1.4382945210793663e-05, "epoch": 0.8800200384069466, "percentage": 94.16, "elapsed_time": "0:06:54", "remaining_time": "0:00:25"}
+{"current_steps": 10550, "total_steps": 11194, "loss": 0.6097, "lr": 1.4104508614417267e-05, "epoch": 0.8808549720297236, "percentage": 94.25, "elapsed_time": "0:08:23", "remaining_time": "0:00:30"}
+{"current_steps": 10560, "total_steps": 11194, "loss": 0.5803, "lr": 1.382221094560512e-05, "epoch": 0.8816899056525006, "percentage": 94.34, "elapsed_time": "0:09:53", "remaining_time": "0:00:35"}
+{"current_steps": 10570, "total_steps": 11194, "loss": 0.6179, "lr": 1.3536321033912506e-05, "epoch": 0.8825248392752776, "percentage": 94.43, "elapsed_time": "0:11:22", "remaining_time": "0:00:40"}
+{"current_steps": 10580, "total_steps": 11194, "loss": 0.6132, "lr": 1.3247111129755609e-05, "epoch": 0.8833597728980546, "percentage": 94.51, "elapsed_time": "0:12:52", "remaining_time": "0:00:44"}
+{"current_steps": 10590, "total_steps": 11194, "loss": 0.5695, "lr": 1.2954856645149868e-05, "epoch": 0.8841947065208315, "percentage": 94.6, "elapsed_time": "0:14:21", "remaining_time": "0:00:49"}
+{"current_steps": 10600, "total_steps": 11194, "loss": 0.6059, "lr": 1.2659835891437553e-05, "epoch": 0.8850296401436086, "percentage": 94.69, "elapsed_time": "0:15:51", "remaining_time": "0:00:53"}
+{"current_steps": 10610, "total_steps": 11194, "loss": 0.5948, "lr": 1.236232981425436e-05, "epoch": 0.8858645737663856, "percentage": 94.78, "elapsed_time": "0:17:20", "remaining_time": "0:00:57"}
+{"current_steps": 10620, "total_steps": 11194, "loss": 0.6181, "lr": 1.2062621725987402e-05, "epoch": 0.8866995073891626, "percentage": 94.87, "elapsed_time": "0:18:49", "remaining_time": "0:01:01"}
+{"current_steps": 10630, "total_steps": 11194, "loss": 0.5956, "lr": 1.1760997035979318e-05, "epoch": 0.8875344410119396, "percentage": 94.96, "elapsed_time": "0:20:19", "remaining_time": "0:01:04"}
+{"current_steps": 10640, "total_steps": 11194, "loss": 0.6072, "lr": 1.1457742978735514e-05, "epoch": 0.8883693746347165, "percentage": 95.05, "elapsed_time": "0:21:48", "remaining_time": "0:01:08"}
+{"current_steps": 10650, "total_steps": 11194, "loss": 0.5847, "lr": 1.1153148340393298e-05, "epoch": 0.8892043082574935, "percentage": 95.14, "elapsed_time": "0:23:18", "remaining_time": "0:01:11"}
+{"current_steps": 10660, "total_steps": 11194, "loss": 0.5935, "lr": 1.0847503183713424e-05, "epoch": 0.8900392418802705, "percentage": 95.23, "elapsed_time": "0:25:25", "remaining_time": "0:01:16"}
+{"current_steps": 10670, "total_steps": 11194, "loss": 0.5598, "lr": 1.0541098571855896e-05, "epoch": 0.8908741755030475, "percentage": 95.32, "elapsed_time": "0:26:54", "remaining_time": "0:01:19"}
+{"current_steps": 10680, "total_steps": 11194, "loss": 0.5757, "lr": 1.0234226291203123e-05, "epoch": 0.8917091091258245, "percentage": 95.41, "elapsed_time": "0:28:24", "remaining_time": "0:01:22"}
+{"current_steps": 10690, "total_steps": 11194, "loss": 0.6076, "lr": 9.927178573494333e-06, "epoch": 0.8925440427486014, "percentage": 95.5, "elapsed_time": "0:29:53", "remaining_time": "0:01:24"}
+{"current_steps": 10700, "total_steps": 11194, "loss": 0.5499, "lr": 9.620247817535902e-06, "epoch": 0.8933789763713785, "percentage": 95.59, "elapsed_time": "0:31:23", "remaining_time": "0:01:26"}
+{"current_steps": 10710, "total_steps": 11194, "loss": 0.5649, "lr": 9.313726310752581e-06, "epoch": 0.8942139099941555, "percentage": 95.68, "elapsed_time": "0:32:52", "remaining_time": "0:01:29"}
+{"current_steps": 10720, "total_steps": 11194, "loss": 0.5775, "lr": 9.00790595084477e-06, "epoch": 0.8950488436169325, "percentage": 95.77, "elapsed_time": "0:34:22", "remaining_time": "0:01:31"}
+{"current_steps": 10730, "total_steps": 11194, "loss": 0.5794, "lr": 8.703077967816944e-06, "epoch": 0.8958837772397095, "percentage": 95.85, "elapsed_time": "0:35:51", "remaining_time": "0:01:33"}
+{"current_steps": 10740, "total_steps": 11194, "loss": 0.5889, "lr": 8.39953264664192e-06, "epoch": 0.8967187108624864, "percentage": 95.94, "elapsed_time": "0:37:21", "remaining_time": "0:01:34"}
+{"current_steps": 10750, "total_steps": 11194, "loss": 0.5924, "lr": 8.097559050825058e-06, "epoch": 0.8975536444852634, "percentage": 96.03, "elapsed_time": "0:38:51", "remaining_time": "0:01:36"}
+{"current_steps": 10760, "total_steps": 11194, "loss": 0.5908, "lr": 7.79744474713166e-06, "epoch": 0.8983885781080404, "percentage": 96.12, "elapsed_time": "0:40:20", "remaining_time": "0:01:37"}
+{"current_steps": 10770, "total_steps": 11194, "loss": 0.5894, "lr": 7.499475531739696e-06, "epoch": 0.8992235117308174, "percentage": 96.21, "elapsed_time": "0:41:50", "remaining_time": "0:01:38"}
+{"current_steps": 10780, "total_steps": 11194, "loss": 0.5969, "lr": 7.203935158078662e-06, "epoch": 0.9000584453535944, "percentage": 96.3, "elapsed_time": "0:43:19", "remaining_time": "0:01:39"}
+{"current_steps": 10790, "total_steps": 11194, "loss": 0.5583, "lr": 6.911105066613709e-06, "epoch": 0.9008933789763713, "percentage": 96.39, "elapsed_time": "0:44:49", "remaining_time": "0:01:40"}
+{"current_steps": 10800, "total_steps": 11194, "loss": 0.5913, "lr": 6.621264116832394e-06, "epoch": 0.9017283125991483, "percentage": 96.48, "elapsed_time": "0:46:18", "remaining_time": "0:01:41"}
+{"current_steps": 10810, "total_steps": 11194, "loss": 0.5793, "lr": 6.334688321689274e-06, "epoch": 0.9025632462219254, "percentage": 96.57, "elapsed_time": "0:47:48", "remaining_time": "0:01:41"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.5779, "lr": 6.051650584761237e-06, "epoch": 0.9033981798447024, "percentage": 96.66, "elapsed_time": "0:49:55", "remaining_time": "0:01:43"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.5792, "lr": 5.772420440363841e-06, "epoch": 0.9042331134674794, "percentage": 96.75, "elapsed_time": "0:51:24", "remaining_time": "0:01:43"}
+{"current_steps": 10820, "total_steps": 11194, "loss": 0.5634, "lr": 6.051650584761237e-06, "epoch": 0.9033981798447024, "percentage": 96.66, "elapsed_time": "0:01:13", "remaining_time": "0:00:02"}
+{"current_steps": 10830, "total_steps": 11194, "loss": 0.5793, "lr": 5.772420440363841e-06, "epoch": 0.9042331134674794, "percentage": 96.75, "elapsed_time": "0:02:42", "remaining_time": "0:00:05"}
+{"current_steps": 10840, "total_steps": 11194, "loss": 0.6141, "lr": 5.497263796876207e-06, "epoch": 0.9050680470902563, "percentage": 96.84, "elapsed_time": "0:04:10", "remaining_time": "0:00:08"}
+{"current_steps": 10850, "total_steps": 11194, "loss": 0.5809, "lr": 5.226442683518813e-06, "epoch": 0.9059029807130333, "percentage": 96.93, "elapsed_time": "0:05:39", "remaining_time": "0:00:10"}
+{"current_steps": 10860, "total_steps": 11194, "loss": 0.5773, "lr": 4.960215000825432e-06, "epoch": 0.9067379143358103, "percentage": 97.02, "elapsed_time": "0:07:07", "remaining_time": "0:00:13"}
+{"current_steps": 10870, "total_steps": 11194, "loss": 0.5832, "lr": 4.698834275046711e-06, "epoch": 0.9075728479585873, "percentage": 97.11, "elapsed_time": "0:08:36", "remaining_time": "0:00:15"}
+{"current_steps": 10880, "total_steps": 11194, "loss": 0.6144, "lr": 4.442549416719423e-06, "epoch": 0.9084077815813643, "percentage": 97.19, "elapsed_time": "0:10:05", "remaining_time": "0:00:17"}
+{"current_steps": 10890, "total_steps": 11194, "loss": 0.599, "lr": 4.1916044836311565e-06, "epoch": 0.9092427152041412, "percentage": 97.28, "elapsed_time": "0:11:33", "remaining_time": "0:00:19"}
+{"current_steps": 10900, "total_steps": 11194, "loss": 0.5804, "lr": 3.94623844840626e-06, "epoch": 0.9100776488269182, "percentage": 97.37, "elapsed_time": "0:13:02", "remaining_time": "0:00:21"}
+{"current_steps": 10910, "total_steps": 11194, "loss": 0.5725, "lr": 3.7066849709343467e-06, "epoch": 0.9109125824496952, "percentage": 97.46, "elapsed_time": "0:14:30", "remaining_time": "0:00:22"}
+{"current_steps": 10920, "total_steps": 11194, "loss": 0.5957, "lr": 3.473172175858059e-06, "epoch": 0.9117475160724723, "percentage": 97.55, "elapsed_time": "0:15:59", "remaining_time": "0:00:24"}
+{"current_steps": 10930, "total_steps": 11194, "loss": 0.5703, "lr": 3.2459224353320046e-06, "epoch": 0.9125824496952493, "percentage": 97.64, "elapsed_time": "0:17:28", "remaining_time": "0:00:25"}
+{"current_steps": 10940, "total_steps": 11194, "loss": 0.5976, "lr": 3.025152157259735e-06, "epoch": 0.9134173833180262, "percentage": 97.73, "elapsed_time": "0:18:56", "remaining_time": "0:00:26"}
+{"current_steps": 10950, "total_steps": 11194, "loss": 0.5907, "lr": 2.811071579210425e-06, "epoch": 0.9142523169408032, "percentage": 97.82, "elapsed_time": "0:20:25", "remaining_time": "0:00:27"}
+{"current_steps": 10960, "total_steps": 11194, "loss": 0.6071, "lr": 2.603884568211513e-06, "epoch": 0.9150872505635802, "percentage": 97.91, "elapsed_time": "0:21:53", "remaining_time": "0:00:28"}
+{"current_steps": 10970, "total_steps": 11194, "loss": 0.5813, "lr": 2.4037884266079434e-06, "epoch": 0.9159221841863572, "percentage": 98.0, "elapsed_time": "0:23:22", "remaining_time": "0:00:28"}
+{"current_steps": 10980, "total_steps": 11194, "loss": 0.5803, "lr": 2.2109737041728955e-06, "epoch": 0.9167571178091342, "percentage": 98.09, "elapsed_time": "0:25:28", "remaining_time": "0:00:29"}
+{"current_steps": 10990, "total_steps": 11194, "loss": 0.6052, "lr": 2.0256240166489454e-06, "epoch": 0.9175920514319111, "percentage": 98.18, "elapsed_time": "0:26:57", "remaining_time": "0:00:30"}
+{"current_steps": 11000, "total_steps": 11194, "loss": 0.5754, "lr": 1.847915870892413e-06, "epoch": 0.9184269850546881, "percentage": 98.27, "elapsed_time": "0:28:25", "remaining_time": "0:00:30"}
+{"current_steps": 11010, "total_steps": 11194, "loss": 0.5366, "lr": 1.6780184967874715e-06, "epoch": 0.9192619186774651, "percentage": 98.36, "elapsed_time": "0:29:54", "remaining_time": "0:00:29"}
+{"current_steps": 11020, "total_steps": 11194, "loss": 0.5975, "lr": 1.5160936860900294e-06, "epoch": 0.9200968523002422, "percentage": 98.45, "elapsed_time": "0:31:22", "remaining_time": "0:00:29"}
+{"current_steps": 11030, "total_steps": 11194, "loss": 0.5952, "lr": 1.3622956383548762e-06, "epoch": 0.9209317859230192, "percentage": 98.53, "elapsed_time": "0:32:51", "remaining_time": "0:00:29"}
+{"current_steps": 11040, "total_steps": 11194, "loss": 0.6047, "lr": 1.216770814092834e-06, "epoch": 0.9217667195457961, "percentage": 98.62, "elapsed_time": "0:34:20", "remaining_time": "0:00:28"}
+{"current_steps": 11050, "total_steps": 11194, "loss": 0.6029, "lr": 1.0796577952977013e-06, "epoch": 0.9226016531685731, "percentage": 98.71, "elapsed_time": "0:35:48", "remaining_time": "0:00:28"}
+{"current_steps": 11060, "total_steps": 11194, "loss": 0.594, "lr": 9.510871534758705e-07, "epoch": 0.9234365867913501, "percentage": 98.8, "elapsed_time": "0:37:17", "remaining_time": "0:00:27"}
+{"current_steps": 11070, "total_steps": 11194, "loss": 0.5881, "lr": 8.311813253042334e-07, "epoch": 0.9242715204141271, "percentage": 98.89, "elapsed_time": "0:38:45", "remaining_time": "0:00:26"}
+{"current_steps": 11080, "total_steps": 11194, "loss": 0.5578, "lr": 7.200544960348308e-07, "epoch": 0.9251064540369041, "percentage": 98.98, "elapsed_time": "0:40:14", "remaining_time": "0:00:24"}
+{"current_steps": 11090, "total_steps": 11194, "loss": 0.5963, "lr": 6.178124907572545e-07, "epoch": 0.925941387659681, "percentage": 99.07, "elapsed_time": "0:41:43", "remaining_time": "0:00:23"}
+{"current_steps": 11100, "total_steps": 11194, "loss": 0.5722, "lr": 5.245526736223528e-07, "epoch": 0.926776321282458, "percentage": 99.16, "elapsed_time": "0:43:11", "remaining_time": "0:00:21"}
+{"current_steps": 11110, "total_steps": 11194, "loss": 0.5657, "lr": 4.403638551232205e-07, "epoch": 0.927611254905235, "percentage": 99.25, "elapsed_time": "0:44:40", "remaining_time": "0:00:20"}
+{"current_steps": 11120, "total_steps": 11194, "loss": 0.584, "lr": 3.6532620752175616e-07, "epoch": 0.928446188528012, "percentage": 99.34, "elapsed_time": "0:46:09", "remaining_time": "0:00:18"}
+{"current_steps": 11130, "total_steps": 11194, "loss": 0.5905, "lr": 2.9951118850133247e-07, "epoch": 0.9292811221507891, "percentage": 99.43, "elapsed_time": "0:47:37", "remaining_time": "0:00:16"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.5859, "lr": 2.429814731182774e-07, "epoch": 0.930116055773566, "percentage": 99.52, "elapsed_time": "0:49:43", "remaining_time": "0:00:14"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.5852, "lr": 1.957908941169813e-07, "epoch": 0.930950989396343, "percentage": 99.61, "elapsed_time": "0:51:12", "remaining_time": "0:00:12"}
+{"current_steps": 11140, "total_steps": 11194, "loss": 0.5859, "lr": 2.429814731182774e-07, "epoch": 0.930116055773566, "percentage": 99.52, "elapsed_time": "0:01:32", "remaining_time": "0:00:00"}
+{"current_steps": 11150, "total_steps": 11194, "loss": 0.5852, "lr": 1.957908941169813e-07, "epoch": 0.930950989396343, "percentage": 99.61, "elapsed_time": "0:03:01", "remaining_time": "0:00:00"}
+{"current_steps": 11160, "total_steps": 11194, "loss": 0.5804, "lr": 1.5798439066545254e-07, "epoch": 0.93178592301912, "percentage": 99.7, "elapsed_time": "0:04:30", "remaining_time": "0:00:00"}
+{"current_steps": 11170, "total_steps": 11194, "loss": 0.5582, "lr": 1.2959796556014471e-07, "epoch": 0.932620856641897, "percentage": 99.79, "elapsed_time": "0:05:59", "remaining_time": "0:00:00"}
+{"current_steps": 11180, "total_steps": 11194, "loss": 0.5803, "lr": 1.1065865094081884e-07, "epoch": 0.933455790264674, "percentage": 99.87, "elapsed_time": "0:07:28", "remaining_time": "0:00:00"}
+{"current_steps": 11190, "total_steps": 11194, "loss": 0.578, "lr": 1.011844825480734e-07, "epoch": 0.9342907238874509, "percentage": 99.96, "elapsed_time": "0:08:56", "remaining_time": "0:00:00"}
+{"current_steps": 11194, "total_steps": 11194, "epoch": 0.9346246973365617, "percentage": 100.0, "elapsed_time": "0:10:10", "remaining_time": "0:00:00"}

g2_1_oldcode_cm_newcode__edit/250M/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.23361442765300158,
+    "total_flos": 1.1366756130249245e+19,
+    "train_loss": 0.06184470440508043,
+    "train_runtime": 2300.2617,
+    "train_samples_per_second": 14.597,
+    "train_steps_per_second": 1.216
+}

g2_1_oldcode_cm_newcode__edit/250M/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}

g2_1_oldcode_cm_newcode__edit/250M/config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 100352
+}

g2_1_oldcode_cm_newcode__edit/250M/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257
+  ],
+  "pad_token_id": 100277,
+  "transformers_version": "4.57.6"
+}

g2_1_oldcode_cm_newcode__edit/250M/model.safetensors.index.json

@@ -0,0 +1,363 @@
+{
+  "metadata": {
+    "total_parameters": 7298617344,
+    "total_size": 14597234688
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

g2_1_oldcode_cm_newcode__edit/250M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}