diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..ecdcc5b06fb9a6799e1087cebda215fb47f1a118 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/checkpoint-1500/config.json b/checkpoint-1500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-1500/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-1500/generation_config.json b/checkpoint-1500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-1500/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-1500/model.safetensors b/checkpoint-1500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c71c12d3fb756fa52a6cf33ad11e23221a15b33c
--- /dev/null
+++ b/checkpoint-1500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e03f7b409c3a292cb0f236a71344ba039503e62c2b9a429be411a734bb56be7f
+size 6425529112
diff --git a/checkpoint-1500/optimizer.pt b/checkpoint-1500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdca2c284310dc407c99080eb3ba58eca142211d
--- /dev/null
+++ b/checkpoint-1500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa02c88a1a885e03f7d46b34773567c9b6d16da58f6fe594bcdf0971a268273c
+size 12851221159
diff --git a/checkpoint-1500/rng_state_0.pth b/checkpoint-1500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3cd72c974e82825e0615a259e4bcca95a3a3273c
--- /dev/null
+++ b/checkpoint-1500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b80829a78a755ba0cf5e985f9212f164229ce25f7c02fa877b4cdebf230922b5
+size 15429
diff --git a/checkpoint-1500/rng_state_1.pth b/checkpoint-1500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2749e3e0a290757087922def08bd7cfa7e218b5e
--- /dev/null
+++ b/checkpoint-1500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c6f4074cb5241eff8959c1f45b18c23c893119f6c03fa2da27359dbe7dba66c
+size 15429
diff --git a/checkpoint-1500/rng_state_2.pth b/checkpoint-1500/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c4dcd559098a83c8196eebb4aa54e5ee6c3f2a8f
--- /dev/null
+++ b/checkpoint-1500/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e84026d823a1c04897f710cd119a65b654c08ad443e97c4486fc7a26f6e64b
+size 15429
diff --git a/checkpoint-1500/rng_state_3.pth b/checkpoint-1500/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..57161b2dc15e410494ead6a7f9ba4524f334abae
--- /dev/null
+++ b/checkpoint-1500/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9b60a41c989678f542cc75416e61e7a43f81d6221d259490134253d1f38a920
+size 15429
diff --git a/checkpoint-1500/scheduler.pt b/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6d1b684d7b8eb3fa89aa62a464980e333c534373
--- /dev/null
+++ b/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64e8b2bdcac5ca2a3c5198f5b93e2fbd722a6494563b05b44a3f9f428774cad4
+size 1465
diff --git a/checkpoint-1500/tokenizer.json b/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-1500/tokenizer_config.json b/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-1500/trainer_state.json b/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e8d7c61958c26af550d776169e06be6cf3431e0b
--- /dev/null
+++ b/checkpoint-1500/trainer_state.json
@@ -0,0 +1,244 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4453125,
+      "learning_rate": 9.8e-06,
+      "loss": 1.4362579345703126,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2353227996826173,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.359375,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1962551879882812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1289045715332031,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1640625,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1314387512207031,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1489389038085938,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.0703125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1160533142089843,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0625,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.040457763671875,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.924019775390625,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8882070922851563,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8971571350097656,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.1015625,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9332583618164062,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.203125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8832476806640625,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9302313232421875,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9123858642578125,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6757525634765625,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4296875,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6670223999023438,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6872428894042969,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6917278289794921,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.25,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6836544036865234,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6740491485595703,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.265625,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6897286987304687,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.53125,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6019991302490234,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.49853515625,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.625,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5089201354980468,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5025421142578125,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5206479263305664,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5091935729980469,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084111785888672,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.522473373413086,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6625649181104538e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-1500/training_args.bin b/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..eb6bffcfc788e1062fb4fc4840d0c771fed34e18
--- /dev/null
+++ b/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9507d637355de62eba1ed595b47f14c9f1b33d51e48dea4351d68be1dc0fe373
+size 5265
diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-2000/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-2000/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-2000/model.safetensors b/checkpoint-2000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5b173df584d3d528988fb8fa7e93c80cb47fb4f0
--- /dev/null
+++ b/checkpoint-2000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fb39373725541de20531b434546ae44e6cc80545d1fc9d856f22204914e984
+size 6425529112
diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab600d32a2a29f47b3ba35313840e74347a2069a
--- /dev/null
+++ b/checkpoint-2000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0960b3ed5c9d7a7e0e469d0963a7f780aedfa6c6fc8c8fc04fbe12bd829792b6
+size 12851221159
diff --git a/checkpoint-2000/rng_state_0.pth b/checkpoint-2000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..afcc75952a72312775719a5b7436d424ac06a7be
--- /dev/null
+++ b/checkpoint-2000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:485c3eb7b8ebe39984117ce19c256db4c11dba6dcd8d32908be6d94ca7a4d529
+size 15429
diff --git a/checkpoint-2000/rng_state_1.pth b/checkpoint-2000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..187216abeb18820101c39ef4d7489769e9c8b018
--- /dev/null
+++ b/checkpoint-2000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7cd0007903b1a0344b4ba578f518e1366c3607febc6b70080704119da49e8fc
+size 15429
diff --git a/checkpoint-2000/rng_state_2.pth b/checkpoint-2000/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cfc3b0628b54f6466425b592bb5d9cd311937f5a
--- /dev/null
+++ b/checkpoint-2000/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a453120913ecd5657ebaf5fb69504cf1a158e4ff4015f66acf4be6e06bc270e0
+size 15429
diff --git a/checkpoint-2000/rng_state_3.pth b/checkpoint-2000/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b116710f2349218d380226e98e466d0cb6e39731
--- /dev/null
+++ b/checkpoint-2000/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c18edf48cc3adf4eb70b31f8e96972c6e25718fd8fcd0866b9ace124a9269d2
+size 15429
diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cda936c081683a1e2e11ccf1e0f73ce3fcaf376b
--- /dev/null
+++ b/checkpoint-2000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f186aef94d351d54ba2d4a2a29efca5c57520f3b20fd57991689d6221f1607f
+size 1465
diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-2000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-2000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..d197fd99a7979a17bbaf7d9036b86d77c3c02f14
--- /dev/null
+++ b/checkpoint-2000/trainer_state.json
@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.333333333333333,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4453125,
+      "learning_rate": 9.8e-06,
+      "loss": 1.4362579345703126,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2353227996826173,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.359375,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1962551879882812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1289045715332031,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1640625,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1314387512207031,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1489389038085938,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.0703125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1160533142089843,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0625,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.040457763671875,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.924019775390625,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8882070922851563,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8971571350097656,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.1015625,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9332583618164062,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.203125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8832476806640625,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9302313232421875,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9123858642578125,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6757525634765625,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4296875,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6670223999023438,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6872428894042969,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6917278289794921,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.25,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6836544036865234,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6740491485595703,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.265625,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6897286987304687,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.53125,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6019991302490234,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.49853515625,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.625,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5089201354980468,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5025421142578125,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5206479263305664,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5091935729980469,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084111785888672,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.522473373413086,
+      "step": 1500
+    },
+    {
+      "epoch": 4.133333333333334,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.5429843301134782e-05,
+      "loss": 0.4226302337646484,
+      "step": 1550
+    },
+    {
+      "epoch": 4.266666666666667,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.4603213532306782e-05,
+      "loss": 0.40151473999023435,
+      "step": 1600
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.3777788790940136e-05,
+      "loss": 0.39469322204589846,
+      "step": 1650
+    },
+    {
+      "epoch": 4.533333333333333,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.2956075864845435e-05,
+      "loss": 0.3755529022216797,
+      "step": 1700
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.21405702691723e-05,
+      "loss": 0.4053420257568359,
+      "step": 1750
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.1333748667611854e-05,
+      "loss": 0.3910691833496094,
+      "step": 1800
+    },
+    {
+      "epoch": 4.933333333333334,
+      "grad_norm": 1.515625,
+      "learning_rate": 1.0538061350850545e-05,
+      "loss": 0.41198493957519533,
+      "step": 1850
+    },
+    {
+      "epoch": 5.066666666666666,
+      "grad_norm": 1.3984375,
+      "learning_rate": 9.755924795117798e-06,
+      "loss": 0.3768700408935547,
+      "step": 1900
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 1.5,
+      "learning_rate": 8.989714323427191e-06,
+      "loss": 0.3425804901123047,
+      "step": 1950
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 1.4140625,
+      "learning_rate": 8.241756891798485e-06,
+      "loss": 0.3398160934448242,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.2167532241472717e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..eb6bffcfc788e1062fb4fc4840d0c771fed34e18
--- /dev/null
+++ b/checkpoint-2000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9507d637355de62eba1ed595b47f14c9f1b33d51e48dea4351d68be1dc0fe373
+size 5265
diff --git a/checkpoint-2500/config.json b/checkpoint-2500/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-2500/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-2500/generation_config.json b/checkpoint-2500/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-2500/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-2500/model.safetensors b/checkpoint-2500/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..aa59e8c6a97e1e4638a4c74bfdaae455b2c38685
--- /dev/null
+++ b/checkpoint-2500/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed27a08098f998beb90f01539b8e9287b3836104dc9aff236e338189998ba5f7
+size 6425529112
diff --git a/checkpoint-2500/optimizer.pt b/checkpoint-2500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e21eb8943d7ded7df001f61c854229d2de5f033
--- /dev/null
+++ b/checkpoint-2500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10d0442d02feafd5414dd9b8da080f6d54a44fdac99ed2b78898bbb4e03346a6
+size 12851221159
diff --git a/checkpoint-2500/rng_state_0.pth b/checkpoint-2500/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d9c6e2fa7a82308b474a1e6c40625921e6fe2151
--- /dev/null
+++ b/checkpoint-2500/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63f8c8ac7017e076bdeab5686ee808dc34f2253a11672f9168162f1c139098a8
+size 15429
diff --git a/checkpoint-2500/rng_state_1.pth b/checkpoint-2500/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b12387d1eece7fdf144946f0cb071ea0a6b446fe
--- /dev/null
+++ b/checkpoint-2500/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da9840d31d966ef4acd9553c28e732a2f97a00ae497036cfaf9f1c9aaf7bad46
+size 15429
diff --git a/checkpoint-2500/rng_state_2.pth b/checkpoint-2500/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..19c020cf6e41677f04cad4042ff026f710bd80b6
--- /dev/null
+++ b/checkpoint-2500/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c0d619a42d9d6b525fc902030d45e64a2643605df5c896eef51533128bd7def
+size 15429
diff --git a/checkpoint-2500/rng_state_3.pth b/checkpoint-2500/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7e98abe464aa16b0e983088fb02aa131798cfe57
--- /dev/null
+++ b/checkpoint-2500/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dcc41326d60bf7e5dae8feb8751b9fe5898a4cea6e4b58c37c2d6f2164d12ea
+size 15429
diff --git a/checkpoint-2500/scheduler.pt b/checkpoint-2500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76b7637cdb3336e7908f4ee1351fcb3e8c0dad8a
--- /dev/null
+++ b/checkpoint-2500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f29dc8ae3cab366b02c0b5bf27f08a3f22095fddf6c5ecc87bdc43329fd253cb
+size 1465
diff --git a/checkpoint-2500/tokenizer.json b/checkpoint-2500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-2500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-2500/tokenizer_config.json b/checkpoint-2500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-2500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-2500/trainer_state.json b/checkpoint-2500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..a360021ae13b7d24794312aa6ebef7cc7b3990a9
--- /dev/null
+++ b/checkpoint-2500/trainer_state.json
@@ -0,0 +1,384 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 6.666666666666667,
+  "eval_steps": 500,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4453125,
+      "learning_rate": 9.8e-06,
+      "loss": 1.4362579345703126,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2353227996826173,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.359375,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1962551879882812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1289045715332031,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1640625,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1314387512207031,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1489389038085938,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.0703125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1160533142089843,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0625,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.040457763671875,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.924019775390625,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8882070922851563,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8971571350097656,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.1015625,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9332583618164062,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.203125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8832476806640625,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9302313232421875,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9123858642578125,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6757525634765625,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4296875,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6670223999023438,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6872428894042969,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6917278289794921,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.25,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6836544036865234,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6740491485595703,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.265625,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6897286987304687,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.53125,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6019991302490234,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.49853515625,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.625,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5089201354980468,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5025421142578125,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5206479263305664,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5091935729980469,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084111785888672,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.522473373413086,
+      "step": 1500
+    },
+    {
+      "epoch": 4.133333333333334,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.5429843301134782e-05,
+      "loss": 0.4226302337646484,
+      "step": 1550
+    },
+    {
+      "epoch": 4.266666666666667,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.4603213532306782e-05,
+      "loss": 0.40151473999023435,
+      "step": 1600
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.3777788790940136e-05,
+      "loss": 0.39469322204589846,
+      "step": 1650
+    },
+    {
+      "epoch": 4.533333333333333,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.2956075864845435e-05,
+      "loss": 0.3755529022216797,
+      "step": 1700
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.21405702691723e-05,
+      "loss": 0.4053420257568359,
+      "step": 1750
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.1333748667611854e-05,
+      "loss": 0.3910691833496094,
+      "step": 1800
+    },
+    {
+      "epoch": 4.933333333333334,
+      "grad_norm": 1.515625,
+      "learning_rate": 1.0538061350850545e-05,
+      "loss": 0.41198493957519533,
+      "step": 1850
+    },
+    {
+      "epoch": 5.066666666666666,
+      "grad_norm": 1.3984375,
+      "learning_rate": 9.755924795117798e-06,
+      "loss": 0.3768700408935547,
+      "step": 1900
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 1.5,
+      "learning_rate": 8.989714323427191e-06,
+      "loss": 0.3425804901123047,
+      "step": 1950
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 1.4140625,
+      "learning_rate": 8.241756891798485e-06,
+      "loss": 0.3398160934448242,
+      "step": 2000
+    },
+    {
+      "epoch": 5.466666666666667,
+      "grad_norm": 1.953125,
+      "learning_rate": 7.5143240223686934e-06,
+      "loss": 0.33743438720703123,
+      "step": 2050
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 1.2421875,
+      "learning_rate": 6.809624904854006e-06,
+      "loss": 0.3318545532226562,
+      "step": 2100
+    },
+    {
+      "epoch": 5.733333333333333,
+      "grad_norm": 1.6171875,
+      "learning_rate": 6.12979968731323e-06,
+      "loss": 0.33765140533447263,
+      "step": 2150
+    },
+    {
+      "epoch": 5.866666666666667,
+      "grad_norm": 1.5703125,
+      "learning_rate": 5.4769129765884716e-06,
+      "loss": 0.34544857025146486,
+      "step": 2200
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 1.4453125,
+      "learning_rate": 4.852947568162011e-06,
+      "loss": 0.34912044525146485,
+      "step": 2250
+    },
+    {
+      "epoch": 6.133333333333334,
+      "grad_norm": 1.7109375,
+      "learning_rate": 4.2597984244716e-06,
+      "loss": 0.32130874633789064,
+      "step": 2300
+    },
+    {
+      "epoch": 6.266666666666667,
+      "grad_norm": 1.4296875,
+      "learning_rate": 3.699266919971811e-06,
+      "loss": 0.32198089599609375,
+      "step": 2350
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.1730553704190283e-06,
+      "loss": 0.3156669425964356,
+      "step": 2400
+    },
+    {
+      "epoch": 6.533333333333333,
+      "grad_norm": 1.5078125,
+      "learning_rate": 2.6827618629944394e-06,
+      "loss": 0.33737548828125,
+      "step": 2450
+    },
+    {
+      "epoch": 6.666666666666667,
+      "grad_norm": 1.4609375,
+      "learning_rate": 2.229875402965799e-06,
+      "loss": 0.31845829010009763,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.7709415301840896e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-2500/training_args.bin b/checkpoint-2500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..eb6bffcfc788e1062fb4fc4840d0c771fed34e18
--- /dev/null
+++ b/checkpoint-2500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9507d637355de62eba1ed595b47f14c9f1b33d51e48dea4351d68be1dc0fe373
+size 5265
diff --git a/checkpoint-3000/config.json b/checkpoint-3000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/checkpoint-3000/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/checkpoint-3000/generation_config.json b/checkpoint-3000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/checkpoint-3000/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/checkpoint-3000/model.safetensors b/checkpoint-3000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..50e50fa642881882a949e4d067fedf91521ce250
--- /dev/null
+++ b/checkpoint-3000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2150100a3b809762228e21f49ec6830e931cc5df587fb905d72a39af8b6388
+size 6425529112
diff --git a/checkpoint-3000/optimizer.pt b/checkpoint-3000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ec9fdb5582db17e9cbf3ac55c4ec1df0e35780e
--- /dev/null
+++ b/checkpoint-3000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae6fc851701134a88146b2d47beaa55d122cb526daaeeb0ddbf387a4181a58b2
+size 12851221159
diff --git a/checkpoint-3000/rng_state_0.pth b/checkpoint-3000/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c2a2270750ad34278e35aa053aac5182f2a93e99
--- /dev/null
+++ b/checkpoint-3000/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccdd04b664fbfa5d05e90c688a91f18c4d07f58893b9b3a4cfad78d78efbc6ac
+size 15429
diff --git a/checkpoint-3000/rng_state_1.pth b/checkpoint-3000/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..560af6f9618784d275690dd7c0725439475c3d26
--- /dev/null
+++ b/checkpoint-3000/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a41e144bee8116d5a47773b5f326ca7bb65df8c457f0c58a2ed8003d7ffcc73
+size 15429
diff --git a/checkpoint-3000/rng_state_2.pth b/checkpoint-3000/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2537b73c8fbe0077e6a758fa28986d271da796bb
--- /dev/null
+++ b/checkpoint-3000/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa38d5d663317603edb36bb18e908202a95a6e6250c06e2bea719fdba77f1277
+size 15429
diff --git a/checkpoint-3000/rng_state_3.pth b/checkpoint-3000/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..61c811fac31215597828fee76a70aef9491d6bcf
--- /dev/null
+++ b/checkpoint-3000/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbe1f72976acfa5e0e28d92c61725f779f476ad95d06ce9b63a6d9978dd7b9ba
+size 15429
diff --git a/checkpoint-3000/scheduler.pt b/checkpoint-3000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e16a36f6ab13567b92cf264e741af8c5490d0bce
--- /dev/null
+++ b/checkpoint-3000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36922d6c505490e075f78f99bc9d4cf97ba7133355550051865a88ede5aaf5ff
+size 1465
diff --git a/checkpoint-3000/tokenizer.json b/checkpoint-3000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-3000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-3000/tokenizer_config.json b/checkpoint-3000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/checkpoint-3000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/checkpoint-3000/trainer_state.json b/checkpoint-3000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..7d967bd6cf989ff4b22816658c1871ebbf59146b
--- /dev/null
+++ b/checkpoint-3000/trainer_state.json
@@ -0,0 +1,454 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 8.0,
+  "eval_steps": 500,
+  "global_step": 3000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 1.4453125,
+      "learning_rate": 9.8e-06,
+      "loss": 1.4362579345703126,
+      "step": 50
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.98e-05,
+      "loss": 1.2353227996826173,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.359375,
+      "learning_rate": 2.98e-05,
+      "loss": 1.1962551879882812,
+      "step": 150
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.0,
+      "learning_rate": 2.997812450643705e-05,
+      "loss": 1.1289045715332031,
+      "step": 200
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.1640625,
+      "learning_rate": 2.991077005423577e-05,
+      "loss": 1.1314387512207031,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.979813208344657e-05,
+      "loss": 1.1489389038085938,
+      "step": 300
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.0703125,
+      "learning_rate": 2.964055267188472e-05,
+      "loss": 1.1160533142089843,
+      "step": 350
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.0625,
+      "learning_rate": 2.943851038304401e-05,
+      "loss": 1.040457763671875,
+      "step": 400
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.9192618812715216e-05,
+      "loss": 0.924019775390625,
+      "step": 450
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.8903624725515786e-05,
+      "loss": 0.8882070922851563,
+      "step": 500
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.140625,
+      "learning_rate": 2.8572405786990293e-05,
+      "loss": 0.8971571350097656,
+      "step": 550
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.1015625,
+      "learning_rate": 2.819996789816898e-05,
+      "loss": 0.9332583618164062,
+      "step": 600
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.203125,
+      "learning_rate": 2.7787442140679422e-05,
+      "loss": 0.8832476806640625,
+      "step": 650
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 1.2578125,
+      "learning_rate": 2.7336081341688787e-05,
+      "loss": 0.9302313232421875,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.15625,
+      "learning_rate": 2.684725626910889e-05,
+      "loss": 0.9123858642578125,
+      "step": 750
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.632245146861912e-05,
+      "loss": 0.6757525634765625,
+      "step": 800
+    },
+    {
+      "epoch": 2.2666666666666666,
+      "grad_norm": 1.4296875,
+      "learning_rate": 2.5763260755150013e-05,
+      "loss": 0.6670223999023438,
+      "step": 850
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 1.34375,
+      "learning_rate": 2.5171382372519647e-05,
+      "loss": 0.6872428894042969,
+      "step": 900
+    },
+    {
+      "epoch": 2.533333333333333,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.4548613835923026e-05,
+      "loss": 0.6917278289794921,
+      "step": 950
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.25,
+      "learning_rate": 2.389684647293749e-05,
+      "loss": 0.6836544036865234,
+      "step": 1000
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.5625,
+      "learning_rate": 2.3218059679623026e-05,
+      "loss": 0.6740491485595703,
+      "step": 1050
+    },
+    {
+      "epoch": 2.9333333333333336,
+      "grad_norm": 1.265625,
+      "learning_rate": 2.2514314909161486e-05,
+      "loss": 0.6897286987304687,
+      "step": 1100
+    },
+    {
+      "epoch": 3.066666666666667,
+      "grad_norm": 1.53125,
+      "learning_rate": 2.1787749411291056e-05,
+      "loss": 0.6019991302490234,
+      "step": 1150
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 1.578125,
+      "learning_rate": 2.1040569741549002e-05,
+      "loss": 0.49853515625,
+      "step": 1200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 1.625,
+      "learning_rate": 2.0275045060035044e-05,
+      "loss": 0.5089201354980468,
+      "step": 1250
+    },
+    {
+      "epoch": 3.466666666666667,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9493500240046727e-05,
+      "loss": 0.5025421142578125,
+      "step": 1300
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.8698308807515613e-05,
+      "loss": 0.5206479263305664,
+      "step": 1350
+    },
+    {
+      "epoch": 3.7333333333333334,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.7891885732687026e-05,
+      "loss": 0.5091935729980469,
+      "step": 1400
+    },
+    {
+      "epoch": 3.8666666666666667,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.7076680095934813e-05,
+      "loss": 0.5084111785888672,
+      "step": 1450
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.6255167649984738e-05,
+      "loss": 0.522473373413086,
+      "step": 1500
+    },
+    {
+      "epoch": 4.133333333333334,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.5429843301134782e-05,
+      "loss": 0.4226302337646484,
+      "step": 1550
+    },
+    {
+      "epoch": 4.266666666666667,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.4603213532306782e-05,
+      "loss": 0.40151473999023435,
+      "step": 1600
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 1.1953125,
+      "learning_rate": 1.3777788790940136e-05,
+      "loss": 0.39469322204589846,
+      "step": 1650
+    },
+    {
+      "epoch": 4.533333333333333,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.2956075864845435e-05,
+      "loss": 0.3755529022216797,
+      "step": 1700
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.21405702691723e-05,
+      "loss": 0.4053420257568359,
+      "step": 1750
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.1333748667611854e-05,
+      "loss": 0.3910691833496094,
+      "step": 1800
+    },
+    {
+      "epoch": 4.933333333333334,
+      "grad_norm": 1.515625,
+      "learning_rate": 1.0538061350850545e-05,
+      "loss": 0.41198493957519533,
+      "step": 1850
+    },
+    {
+      "epoch": 5.066666666666666,
+      "grad_norm": 1.3984375,
+      "learning_rate": 9.755924795117798e-06,
+      "loss": 0.3768700408935547,
+      "step": 1900
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 1.5,
+      "learning_rate": 8.989714323427191e-06,
+      "loss": 0.3425804901123047,
+      "step": 1950
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 1.4140625,
+      "learning_rate": 8.241756891798485e-06,
+      "loss": 0.3398160934448242,
+      "step": 2000
+    },
+    {
+      "epoch": 5.466666666666667,
+      "grad_norm": 1.953125,
+      "learning_rate": 7.5143240223686934e-06,
+      "loss": 0.33743438720703123,
+      "step": 2050
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 1.2421875,
+      "learning_rate": 6.809624904854006e-06,
+      "loss": 0.3318545532226562,
+      "step": 2100
+    },
+    {
+      "epoch": 5.733333333333333,
+      "grad_norm": 1.6171875,
+      "learning_rate": 6.12979968731323e-06,
+      "loss": 0.33765140533447263,
+      "step": 2150
+    },
+    {
+      "epoch": 5.866666666666667,
+      "grad_norm": 1.5703125,
+      "learning_rate": 5.4769129765884716e-06,
+      "loss": 0.34544857025146486,
+      "step": 2200
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 1.4453125,
+      "learning_rate": 4.852947568162011e-06,
+      "loss": 0.34912044525146485,
+      "step": 2250
+    },
+    {
+      "epoch": 6.133333333333334,
+      "grad_norm": 1.7109375,
+      "learning_rate": 4.2597984244716e-06,
+      "loss": 0.32130874633789064,
+      "step": 2300
+    },
+    {
+      "epoch": 6.266666666666667,
+      "grad_norm": 1.4296875,
+      "learning_rate": 3.699266919971811e-06,
+      "loss": 0.32198089599609375,
+      "step": 2350
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.1730553704190283e-06,
+      "loss": 0.3156669425964356,
+      "step": 2400
+    },
+    {
+      "epoch": 6.533333333333333,
+      "grad_norm": 1.5078125,
+      "learning_rate": 2.6827618629944394e-06,
+      "loss": 0.33737548828125,
+      "step": 2450
+    },
+    {
+      "epoch": 6.666666666666667,
+      "grad_norm": 1.4609375,
+      "learning_rate": 2.229875402965799e-06,
+      "loss": 0.31845829010009763,
+      "step": 2500
+    },
+    {
+      "epoch": 6.8,
+      "grad_norm": 1.265625,
+      "learning_rate": 1.8157713916273593e-06,
+      "loss": 0.3165144920349121,
+      "step": 2550
+    },
+    {
+      "epoch": 6.933333333333334,
+      "grad_norm": 1.5390625,
+      "learning_rate": 1.4417074492513516e-06,
+      "loss": 0.3110966682434082,
+      "step": 2600
+    },
+    {
+      "epoch": 7.066666666666666,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.10881959573655e-06,
+      "loss": 0.30882575988769534,
+      "step": 2650
+    },
+    {
+      "epoch": 7.2,
+      "grad_norm": 1.546875,
+      "learning_rate": 8.181188005531748e-07,
+      "loss": 0.320262451171875,
+      "step": 2700
+    },
+    {
+      "epoch": 7.333333333333333,
+      "grad_norm": 1.3984375,
+      "learning_rate": 5.704879124618434e-07,
+      "loss": 0.3117942237854004,
+      "step": 2750
+    },
+    {
+      "epoch": 7.466666666666667,
+      "grad_norm": 1.7734375,
+      "learning_rate": 3.666789783308955e-07,
+      "loss": 0.307753791809082,
+      "step": 2800
+    },
+    {
+      "epoch": 7.6,
+      "grad_norm": 1.21875,
+      "learning_rate": 2.073109591947725e-07,
+      "loss": 0.32146541595458983,
+      "step": 2850
+    },
+    {
+      "epoch": 7.733333333333333,
+      "grad_norm": 1.515625,
+      "learning_rate": 9.286785048970181e-08,
+      "loss": 0.31243988037109377,
+      "step": 2900
+    },
+    {
+      "epoch": 7.866666666666667,
+      "grad_norm": 1.5390625,
+      "learning_rate": 2.3697212175460835e-08,
+      "loss": 0.31487451553344725,
+      "step": 2950
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 1.40625,
+      "learning_rate": 9.113207203870211e-12,
+      "loss": 0.32478099822998047,
+      "step": 3000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.3251298362209075e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-3000/training_args.bin b/checkpoint-3000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..eb6bffcfc788e1062fb4fc4840d0c771fed34e18
--- /dev/null
+++ b/checkpoint-3000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9507d637355de62eba1ed595b47f14c9f1b33d51e48dea4351d68be1dc0fe373
+size 5265
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..703de243df321b85c26832f48c923eb928e5ea16
--- /dev/null
+++ b/config.json
@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd5c214ba6644c149e92991d8f72378202332d1
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..50e50fa642881882a949e4d067fedf91521ce250
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2150100a3b809762228e21f49ec6830e931cc5df587fb905d72a39af8b6388
+size 6425529112
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35f0a8972c032adea64d32508952119451312298
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..eb6bffcfc788e1062fb4fc4840d0c771fed34e18
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9507d637355de62eba1ed595b47f14c9f1b33d51e48dea4351d68be1dc0fe373
+size 5265