diff --git a/.gitattributes b/.gitattributes
index 5dd9d48057b35ecb57aedf1dadf40fcfc2e28a57..a7ecfd7073d4535cd5c23d12a6aece40cd962572 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 wandb/run-20260226_135602-696nxyfr/run-696nxyfr.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20260226_153026-trcpjlfd/run-trcpjlfd.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20260319_063518-29lbcxak/run-29lbcxak.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20260319_091054-lisp43b6/run-lisp43b6.wandb filter=lfs diff=lfs merge=lfs -text
diff --git a/checkpoints/metadata_000000016384.json b/checkpoints/metadata_000000016384.json
new file mode 100644
index 0000000000000000000000000000000000000000..48bb1cfe0b90b2ec9afdeff0c3bcd5a30959aa12
--- /dev/null
+++ b/checkpoints/metadata_000000016384.json
@@ -0,0 +1 @@
+{"step": 1, "tokens_seen": 16384, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.505733489990234}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000163840.json b/checkpoints/metadata_000000163840.json
new file mode 100644
index 0000000000000000000000000000000000000000..533c8aac6177d798e3a258832bbd4f73d336e002
--- /dev/null
+++ b/checkpoints/metadata_000000163840.json
@@ -0,0 +1 @@
+{"step": 10, "tokens_seen": 163840, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.493924168611423}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000180224.json b/checkpoints/metadata_000000180224.json
new file mode 100644
index 0000000000000000000000000000000000000000..7b397e294778c65d2f12b31594e1f97bf9b64f7e
--- /dev/null
+++ b/checkpoints/metadata_000000180224.json
@@ -0,0 +1 @@
+{"step": 11, "tokens_seen": 180224, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.490391682372696}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000212992.json b/checkpoints/metadata_000000212992.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca8c8ab1e85addbfb69b2c69e7d71f4fa8f5450a
--- /dev/null
+++ b/checkpoints/metadata_000000212992.json
@@ -0,0 +1 @@
+{"step": 13, "tokens_seen": 212992, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.481672404723923}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000229376.json b/checkpoints/metadata_000000229376.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e1ef3ba01b18246848e7aef3dac21efdd3177cf
--- /dev/null
+++ b/checkpoints/metadata_000000229376.json
@@ -0,0 +1 @@
+{"step": 14, "tokens_seen": 229376, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.476375524731859}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000245760.json b/checkpoints/metadata_000000245760.json
new file mode 100644
index 0000000000000000000000000000000000000000..a72c7c31a7a69df87240a3c84425c28d6c22ba41
--- /dev/null
+++ b/checkpoints/metadata_000000245760.json
@@ -0,0 +1 @@
+{"step": 15, "tokens_seen": 245760, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.470667002486248}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000278528.json b/checkpoints/metadata_000000278528.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca802f2ea7f8aedf9c5851c335ce37a64a08b63a
--- /dev/null
+++ b/checkpoints/metadata_000000278528.json
@@ -0,0 +1 @@
+{"step": 17, "tokens_seen": 278528, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.458815586524315}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000311296.json b/checkpoints/metadata_000000311296.json
new file mode 100644
index 0000000000000000000000000000000000000000..6a81c1eae68dbab4043b1199e3ff5ee1bf97c450
--- /dev/null
+++ b/checkpoints/metadata_000000311296.json
@@ -0,0 +1 @@
+{"step": 19, "tokens_seen": 311296, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.445611761004885}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000344064.json b/checkpoints/metadata_000000344064.json
new file mode 100644
index 0000000000000000000000000000000000000000..e0ffa077df29118b57440884f7f3113fd0e0268a
--- /dev/null
+++ b/checkpoints/metadata_000000344064.json
@@ -0,0 +1 @@
+{"step": 21, "tokens_seen": 344064, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.432933477715588}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000376832.json b/checkpoints/metadata_000000376832.json
new file mode 100644
index 0000000000000000000000000000000000000000..b00d879f37665866a6879861d20c2fa410b5c391
--- /dev/null
+++ b/checkpoints/metadata_000000376832.json
@@ -0,0 +1 @@
+{"step": 23, "tokens_seen": 376832, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.417946574504807}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000409600.json b/checkpoints/metadata_000000409600.json
new file mode 100644
index 0000000000000000000000000000000000000000..d844fd2e295684aa038cf098b3b21db05b053719
--- /dev/null
+++ b/checkpoints/metadata_000000409600.json
@@ -0,0 +1 @@
+{"step": 25, "tokens_seen": 409600, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.401332998215223}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000458752.json b/checkpoints/metadata_000000458752.json
index 328e5625da713d190d238b090457d5f46289fda4..b21c924d5f90feb3306d0f8f66bdd96f8c932f05 100644
--- a/checkpoints/metadata_000000458752.json
+++ b/checkpoints/metadata_000000458752.json
@@ -1 +1 @@
-{"step": 14, "tokens_seen": 458752, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.423034213406622}
\ No newline at end of file
+{"step": 28, "tokens_seen": 458752, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.378429363888989}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000491520.json b/checkpoints/metadata_000000491520.json
index f1478ad8c12500a120427dc5441980a09c041a05..47e03e9231ba61813ab4e81cbf0d11385ccb01c8 100644
--- a/checkpoints/metadata_000000491520.json
+++ b/checkpoints/metadata_000000491520.json
@@ -1 +1 @@
-{"step": 15, "tokens_seen": 491520, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.416432102321139}
\ No newline at end of file
+{"step": 30, "tokens_seen": 491520, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.362496783724538}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000557056.json b/checkpoints/metadata_000000557056.json
index c75e35d7689d2ca0be22cbcb0ace945ee405ddfc..9b147ed4b6bdf4854aed0783617384fa672156d1 100644
--- a/checkpoints/metadata_000000557056.json
+++ b/checkpoints/metadata_000000557056.json
@@ -1 +1 @@
-{"step": 17, "tokens_seen": 557056, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.401797919113}
\ No newline at end of file
+{"step": 34, "tokens_seen": 557056, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.329005356493877}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000606208.json b/checkpoints/metadata_000000606208.json
new file mode 100644
index 0000000000000000000000000000000000000000..990a0da5174b03d8f810ba64b1aadd519c616b90
--- /dev/null
+++ b/checkpoints/metadata_000000606208.json
@@ -0,0 +1 @@
+{"step": 37, "tokens_seen": 606208, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.304170555971416}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000671744.json b/checkpoints/metadata_000000671744.json
new file mode 100644
index 0000000000000000000000000000000000000000..611bd205cdf41cced817bbb021976d5cd91c9cf9
--- /dev/null
+++ b/checkpoints/metadata_000000671744.json
@@ -0,0 +1 @@
+{"step": 41, "tokens_seen": 671744, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.270269580133991}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000737280.json b/checkpoints/metadata_000000737280.json
new file mode 100644
index 0000000000000000000000000000000000000000..a361f6e817f5be444e51c9dabc32180cfd31cfd8
--- /dev/null
+++ b/checkpoints/metadata_000000737280.json
@@ -0,0 +1 @@
+{"step": 45, "tokens_seen": 737280, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.232169804358817}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000802816.json b/checkpoints/metadata_000000802816.json
new file mode 100644
index 0000000000000000000000000000000000000000..26fce9b92583b975c8e2cafdb7a6b5c969348422
--- /dev/null
+++ b/checkpoints/metadata_000000802816.json
@@ -0,0 +1 @@
+{"step": 49, "tokens_seen": 802816, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.19356061807453}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000884736.json b/checkpoints/metadata_000000884736.json
new file mode 100644
index 0000000000000000000000000000000000000000..105584d640bbfb281c920ce4c18048843ff81f72
--- /dev/null
+++ b/checkpoints/metadata_000000884736.json
@@ -0,0 +1 @@
+{"step": 54, "tokens_seen": 884736, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.140228017963985}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000983040.json b/checkpoints/metadata_000000983040.json
index f387e3860729f218a742964f5aa838aff338e355..bc010ed384e2e1f4601abcdd8b9cae9018ae23b4 100644
--- a/checkpoints/metadata_000000983040.json
+++ b/checkpoints/metadata_000000983040.json
@@ -1 +1 @@
-{"step": 30, "tokens_seen": 983040, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.292225877179932}
\ No newline at end of file
+{"step": 60, "tokens_seen": 983040, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 10.066979510761639}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001081344.json b/checkpoints/metadata_000001081344.json
new file mode 100644
index 0000000000000000000000000000000000000000..8ec6392db8b15fb10110ac3c2032de4a1b242edc
--- /dev/null
+++ b/checkpoints/metadata_000001081344.json
@@ -0,0 +1 @@
+{"step": 66, "tokens_seen": 1081344, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.987235387589818}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001179648.json b/checkpoints/metadata_000001179648.json
new file mode 100644
index 0000000000000000000000000000000000000000..571d1b4b2d776655f4cc69b813594ca5fd0f6e5b
--- /dev/null
+++ b/checkpoints/metadata_000001179648.json
@@ -0,0 +1 @@
+{"step": 72, "tokens_seen": 1179648, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.909608448693483}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001310720.json b/checkpoints/metadata_000001310720.json
new file mode 100644
index 0000000000000000000000000000000000000000..3eaf4d3f130350b9b817a064023fe8b9d7cd333c
--- /dev/null
+++ b/checkpoints/metadata_000001310720.json
@@ -0,0 +1 @@
+{"step": 80, "tokens_seen": 1310720, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.795492891785779}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001441792.json b/checkpoints/metadata_000001441792.json
new file mode 100644
index 0000000000000000000000000000000000000000..39af97b824e5c4359a5c466839cb9b00b6f6e1ce
--- /dev/null
+++ b/checkpoints/metadata_000001441792.json
@@ -0,0 +1 @@
+{"step": 88, "tokens_seen": 1441792, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.673737053513918}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001589248.json b/checkpoints/metadata_000001589248.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7dfd3761c1bf5a1159874bf8dd2d7556e65d16c
--- /dev/null
+++ b/checkpoints/metadata_000001589248.json
@@ -0,0 +1 @@
+{"step": 97, "tokens_seen": 1589248, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.525613877229812}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001736704.json b/checkpoints/metadata_000001736704.json
new file mode 100644
index 0000000000000000000000000000000000000000..a84836aad1e86c70571cb5ad0312bfac09682eed
--- /dev/null
+++ b/checkpoints/metadata_000001736704.json
@@ -0,0 +1 @@
+{"step": 106, "tokens_seen": 1736704, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.376257246466892}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001916928.json b/checkpoints/metadata_000001916928.json
new file mode 100644
index 0000000000000000000000000000000000000000..d876fd7045434a37b4deee7af826dfcdea0256b6
--- /dev/null
+++ b/checkpoints/metadata_000001916928.json
@@ -0,0 +1 @@
+{"step": 117, "tokens_seen": 1916928, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 9.183480711423709}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002113536.json b/checkpoints/metadata_000002113536.json
new file mode 100644
index 0000000000000000000000000000000000000000..5fbff4ad3781b3e900a5d0441fa8b069285bab3e
--- /dev/null
+++ b/checkpoints/metadata_000002113536.json
@@ -0,0 +1 @@
+{"step": 129, "tokens_seen": 2113536, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 8.963223765757022}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002326528.json b/checkpoints/metadata_000002326528.json
new file mode 100644
index 0000000000000000000000000000000000000000..6a3f63ed823c85cd48168fcd3dcd47446caf8f4b
--- /dev/null
+++ b/checkpoints/metadata_000002326528.json
@@ -0,0 +1 @@
+{"step": 142, "tokens_seen": 2326528, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 8.723841847434839}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002555904.json b/checkpoints/metadata_000002555904.json
new file mode 100644
index 0000000000000000000000000000000000000000..3d279ff3f546f619c9a6dcdeae89dc8b8711de38
--- /dev/null
+++ b/checkpoints/metadata_000002555904.json
@@ -0,0 +1 @@
+{"step": 156, "tokens_seen": 2555904, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 8.475664676359377}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002801664.json b/checkpoints/metadata_000002801664.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0432e91257eef4fc9da3c4ebda21b94274cbf87
--- /dev/null
+++ b/checkpoints/metadata_000002801664.json
@@ -0,0 +1 @@
+{"step": 171, "tokens_seen": 2801664, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 8.223454781131126}
\ No newline at end of file
diff --git a/checkpoints/metadata_000003096576.json b/checkpoints/metadata_000003096576.json
new file mode 100644
index 0000000000000000000000000000000000000000..302d97c7ef4407b9b2992daa426bc14ef638dc92
--- /dev/null
+++ b/checkpoints/metadata_000003096576.json
@@ -0,0 +1 @@
+{"step": 189, "tokens_seen": 3096576, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 7.956106340993302}
\ No newline at end of file
diff --git a/checkpoints/metadata_000003391488.json b/checkpoints/metadata_000003391488.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0cde0b0d05300f56fbe06ed87182d269ff212bf
--- /dev/null
+++ b/checkpoints/metadata_000003391488.json
@@ -0,0 +1 @@
+{"step": 207, "tokens_seen": 3391488, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 7.719086985354264}
\ No newline at end of file
diff --git a/checkpoints/metadata_000003735552.json b/checkpoints/metadata_000003735552.json
new file mode 100644
index 0000000000000000000000000000000000000000..6b1eede87f36a40208d8dba52832916813c56f9b
--- /dev/null
+++ b/checkpoints/metadata_000003735552.json
@@ -0,0 +1 @@
+{"step": 228, "tokens_seen": 3735552, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 7.4802159549343985}
\ No newline at end of file
diff --git a/checkpoints/metadata_000004112384.json b/checkpoints/metadata_000004112384.json
new file mode 100644
index 0000000000000000000000000000000000000000..46985c5f4cdf97d31680d83de9d345221661197b
--- /dev/null
+++ b/checkpoints/metadata_000004112384.json
@@ -0,0 +1 @@
+{"step": 251, "tokens_seen": 4112384, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 7.231959520142127}
\ No newline at end of file
diff --git a/checkpoints/metadata_000004521984.json b/checkpoints/metadata_000004521984.json
new file mode 100644
index 0000000000000000000000000000000000000000..55bad3aae270a20811d0d3f0205469e306cdf3d0
--- /dev/null
+++ b/checkpoints/metadata_000004521984.json
@@ -0,0 +1 @@
+{"step": 276, "tokens_seen": 4521984, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 6.984745759670721}
\ No newline at end of file
diff --git a/checkpoints/metadata_000004980736.json b/checkpoints/metadata_000004980736.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a51b94001a56fa8bcad11373c941a0bbbceee3d
--- /dev/null
+++ b/checkpoints/metadata_000004980736.json
@@ -0,0 +1 @@
+{"step": 304, "tokens_seen": 4980736, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 6.762465496803143}
\ No newline at end of file
diff --git a/checkpoints/metadata_000005472256.json b/checkpoints/metadata_000005472256.json
new file mode 100644
index 0000000000000000000000000000000000000000..52db140d64d5ee4e715206dd866ebcb8699b6170
--- /dev/null
+++ b/checkpoints/metadata_000005472256.json
@@ -0,0 +1 @@
+{"step": 334, "tokens_seen": 5472256, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 6.567225124024005}
\ No newline at end of file
diff --git a/checkpoints/metadata_000006029312.json b/checkpoints/metadata_000006029312.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c762c610773aed01e832a6defb93f2faa7bf984
--- /dev/null
+++ b/checkpoints/metadata_000006029312.json
@@ -0,0 +1 @@
+{"step": 368, "tokens_seen": 6029312, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 6.373919580653849}
\ No newline at end of file
diff --git a/checkpoints/metadata_000006635520.json b/checkpoints/metadata_000006635520.json
new file mode 100644
index 0000000000000000000000000000000000000000..82e4fb95092970d0557e7b47e52630b616ab6834
--- /dev/null
+++ b/checkpoints/metadata_000006635520.json
@@ -0,0 +1 @@
+{"step": 405, "tokens_seen": 6635520, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 6.207916381446153}
\ No newline at end of file
diff --git a/checkpoints/metadata_000007290880.json b/checkpoints/metadata_000007290880.json
new file mode 100644
index 0000000000000000000000000000000000000000..a7e28a6f0f5822fa480888872e6a6098e078a0f3
--- /dev/null
+++ b/checkpoints/metadata_000007290880.json
@@ -0,0 +1 @@
+{"step": 445, "tokens_seen": 7290880, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 6.053699377371957}
\ No newline at end of file
diff --git a/checkpoints/metadata_000008028160.json b/checkpoints/metadata_000008028160.json
new file mode 100644
index 0000000000000000000000000000000000000000..3563a82813f8158c85cf0a47268366e887e55f50
--- /dev/null
+++ b/checkpoints/metadata_000008028160.json
@@ -0,0 +1 @@
+{"step": 490, "tokens_seen": 8028160, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.9365480947589395}
\ No newline at end of file
diff --git a/checkpoints/metadata_000008192000.json b/checkpoints/metadata_000008192000.json
new file mode 100644
index 0000000000000000000000000000000000000000..e8ba700048ada00a0d5e0d3c95b0ef374d6015e4
--- /dev/null
+++ b/checkpoints/metadata_000008192000.json
@@ -0,0 +1 @@
+{"step": 500, "tokens_seen": 8192000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.909376017887369}
\ No newline at end of file
diff --git a/checkpoints/metadata_000008830976.json b/checkpoints/metadata_000008830976.json
new file mode 100644
index 0000000000000000000000000000000000000000..d04d58b11a36d808d863aa5282460db2bea1a9df
--- /dev/null
+++ b/checkpoints/metadata_000008830976.json
@@ -0,0 +1 @@
+{"step": 539, "tokens_seen": 8830976, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.812382958695373}
\ No newline at end of file
diff --git a/checkpoints/metadata_000009715712.json b/checkpoints/metadata_000009715712.json
new file mode 100644
index 0000000000000000000000000000000000000000..31c8e1b566fa6fd86c4896f61ffb3ff40a7e2e5e
--- /dev/null
+++ b/checkpoints/metadata_000009715712.json
@@ -0,0 +1 @@
+{"step": 593, "tokens_seen": 9715712, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.699187932751524}
\ No newline at end of file
diff --git a/checkpoints/metadata_000010682368.json b/checkpoints/metadata_000010682368.json
new file mode 100644
index 0000000000000000000000000000000000000000..8cce1f9cc3810fd10c0de294a56b864ec536b6c0
--- /dev/null
+++ b/checkpoints/metadata_000010682368.json
@@ -0,0 +1 @@
+{"step": 652, "tokens_seen": 10682368, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.591649974413007}
\ No newline at end of file
diff --git a/checkpoints/metadata_000011747328.json b/checkpoints/metadata_000011747328.json
new file mode 100644
index 0000000000000000000000000000000000000000..47ed853b172f1864061a337577cfb235fabadc1a
--- /dev/null
+++ b/checkpoints/metadata_000011747328.json
@@ -0,0 +1 @@
+{"step": 717, "tokens_seen": 11747328, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.497387630827847}
\ No newline at end of file
diff --git a/checkpoints/metadata_000012926976.json b/checkpoints/metadata_000012926976.json
new file mode 100644
index 0000000000000000000000000000000000000000..bc9c06ffa4170e3a08e03a30590429190220ba32
--- /dev/null
+++ b/checkpoints/metadata_000012926976.json
@@ -0,0 +1 @@
+{"step": 789, "tokens_seen": 12926976, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.431674134461605}
\ No newline at end of file
diff --git a/checkpoints/metadata_000014221312.json b/checkpoints/metadata_000014221312.json
new file mode 100644
index 0000000000000000000000000000000000000000..889e6cfab56ce8f2e7a6cd120aefa78e8174e221
--- /dev/null
+++ b/checkpoints/metadata_000014221312.json
@@ -0,0 +1 @@
+{"step": 868, "tokens_seen": 14221312, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.3314745516948605}
\ No newline at end of file
diff --git a/checkpoints/metadata_000015646720.json b/checkpoints/metadata_000015646720.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ad889921c04ec8ef4433c24c15719c60a85c339
--- /dev/null
+++ b/checkpoints/metadata_000015646720.json
@@ -0,0 +1 @@
+{"step": 955, "tokens_seen": 15646720, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.265000199510023}
\ No newline at end of file
diff --git a/checkpoints/metadata_000016384000.json b/checkpoints/metadata_000016384000.json
index 0a273985d4c41512db864ec5f11a7e5c0869d76a..1e8d6ef2970999490a1325bdb23520a49a5fe9b3 100644
--- a/checkpoints/metadata_000016384000.json
+++ b/checkpoints/metadata_000016384000.json
@@ -1 +1 @@
-{"step": 500, "tokens_seen": 16384000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 5.468160554157632}
\ No newline at end of file
+{"step": 1000, "tokens_seen": 16384000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.2575255807637165}
\ No newline at end of file
diff --git a/checkpoints/metadata_000017219584.json b/checkpoints/metadata_000017219584.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee473e69d4244de77632eb6f4275d5a7c428c038
--- /dev/null
+++ b/checkpoints/metadata_000017219584.json
@@ -0,0 +1 @@
+{"step": 1051, "tokens_seen": 17219584, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.211203543838707}
\ No newline at end of file
diff --git a/checkpoints/metadata_000018939904.json b/checkpoints/metadata_000018939904.json
new file mode 100644
index 0000000000000000000000000000000000000000..9d11ab12b4afc0a23a542532eeb4618ebd120719
--- /dev/null
+++ b/checkpoints/metadata_000018939904.json
@@ -0,0 +1 @@
+{"step": 1156, "tokens_seen": 18939904, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.112676468529284}
\ No newline at end of file
diff --git a/checkpoints/metadata_000020824064.json b/checkpoints/metadata_000020824064.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f568ea0c53d9bbd0e84d13d4f9b0eb999322c67
--- /dev/null
+++ b/checkpoints/metadata_000020824064.json
@@ -0,0 +1 @@
+{"step": 1271, "tokens_seen": 20824064, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.047554013631982}
\ No newline at end of file
diff --git a/checkpoints/metadata_000022921216.json b/checkpoints/metadata_000022921216.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7985713eaf1566e41a96f9ed03b63f491423ee4
--- /dev/null
+++ b/checkpoints/metadata_000022921216.json
@@ -0,0 +1 @@
+{"step": 1399, "tokens_seen": 22921216, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 5.00726851939288}
\ No newline at end of file
diff --git a/checkpoints/metadata_000024576000.json b/checkpoints/metadata_000024576000.json
new file mode 100644
index 0000000000000000000000000000000000000000..281fb79a3766caaa5965d526a56094b171c5f98e
--- /dev/null
+++ b/checkpoints/metadata_000024576000.json
@@ -0,0 +1 @@
+{"step": 1500, "tokens_seen": 24576000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.930878393785328}
\ No newline at end of file
diff --git a/checkpoints/metadata_000025198592.json b/checkpoints/metadata_000025198592.json
new file mode 100644
index 0000000000000000000000000000000000000000..b5aa4e705144c3876ee506c82f3bc32b474291e0
--- /dev/null
+++ b/checkpoints/metadata_000025198592.json
@@ -0,0 +1 @@
+{"step": 1538, "tokens_seen": 25198592, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.928526568173498}
\ No newline at end of file
diff --git a/checkpoints/metadata_000027721728.json b/checkpoints/metadata_000027721728.json
new file mode 100644
index 0000000000000000000000000000000000000000..2317e2595c209a7b3af7237d522f8ea22d8278b1
--- /dev/null
+++ b/checkpoints/metadata_000027721728.json
@@ -0,0 +1 @@
+{"step": 1692, "tokens_seen": 27721728, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.856798739179598}
\ No newline at end of file
diff --git a/checkpoints/metadata_000030507008.json b/checkpoints/metadata_000030507008.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c9773d66532e47f40340a6413524e01e285cba6
--- /dev/null
+++ b/checkpoints/metadata_000030507008.json
@@ -0,0 +1 @@
+{"step": 1862, "tokens_seen": 30507008, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.778501192972618}
\ No newline at end of file
diff --git a/checkpoints/metadata_000032768000.json b/checkpoints/metadata_000032768000.json
index f6af140337b60e6ed11b0468d888e64445a3f7f7..354e8a52d984dce33dfa2a43303cc014cb2c4783 100644
--- a/checkpoints/metadata_000032768000.json
+++ b/checkpoints/metadata_000032768000.json
@@ -1 +1 @@
-{"step": 1000, "tokens_seen": 32768000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.743142605580698}
\ No newline at end of file
+{"step": 2000, "tokens_seen": 32768000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.695193499554638}
\ No newline at end of file
diff --git a/checkpoints/metadata_000033554432.json b/checkpoints/metadata_000033554432.json
new file mode 100644
index 0000000000000000000000000000000000000000..fcb60c5f373a86495114663315863f2e487d9028
--- /dev/null
+++ b/checkpoints/metadata_000033554432.json
@@ -0,0 +1 @@
+{"step": 2048, "tokens_seen": 33554432, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.660357450029771}
\ No newline at end of file
diff --git a/checkpoints/metadata_000036913152.json b/checkpoints/metadata_000036913152.json
new file mode 100644
index 0000000000000000000000000000000000000000..36c733ab953c368f48bd2a2f0c4beae75ecc5481
--- /dev/null
+++ b/checkpoints/metadata_000036913152.json
@@ -0,0 +1 @@
+{"step": 2253, "tokens_seen": 36913152, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.554995683376985}
\ No newline at end of file
diff --git a/checkpoints/metadata_000040599552.json b/checkpoints/metadata_000040599552.json
new file mode 100644
index 0000000000000000000000000000000000000000..18d8064b4bbfb39f333b941224bb8fd092cc46af
--- /dev/null
+++ b/checkpoints/metadata_000040599552.json
@@ -0,0 +1 @@
+{"step": 2478, "tokens_seen": 40599552, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.455592278749598}
\ No newline at end of file
diff --git a/checkpoints/metadata_000040960000.json b/checkpoints/metadata_000040960000.json
new file mode 100644
index 0000000000000000000000000000000000000000..8b1ec33b5f988db5769c8c8134eab03cc36221fb
--- /dev/null
+++ b/checkpoints/metadata_000040960000.json
@@ -0,0 +1 @@
+{"step": 2500, "tokens_seen": 40960000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.445156686324258}
\ No newline at end of file
diff --git a/checkpoints/metadata_000044662784.json b/checkpoints/metadata_000044662784.json
new file mode 100644
index 0000000000000000000000000000000000000000..58940f425427476f77a4d5794d1f9e460eddf223
--- /dev/null
+++ b/checkpoints/metadata_000044662784.json
@@ -0,0 +1 @@
+{"step": 2726, "tokens_seen": 44662784, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.327059595721218}
\ No newline at end of file
diff --git a/checkpoints/metadata_000049135616.json b/checkpoints/metadata_000049135616.json
new file mode 100644
index 0000000000000000000000000000000000000000..0341db29b6b68a4062ade72e6faaac201708b8f7
--- /dev/null
+++ b/checkpoints/metadata_000049135616.json
@@ -0,0 +1 @@
+{"step": 2999, "tokens_seen": 49135616, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.2469098989500225}
\ No newline at end of file
diff --git a/checkpoints/metadata_000049152000.json b/checkpoints/metadata_000049152000.json
index cd0558d252830ccd2eebc692f23f807f6b94b1ce..9c14a98683104a146a53093fc45286cfe5abf78f 100644
--- a/checkpoints/metadata_000049152000.json
+++ b/checkpoints/metadata_000049152000.json
@@ -1 +1 @@
-{"step": 1500, "tokens_seen": 49152000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 4.300045652251939}
\ No newline at end of file
+{"step": 3000, "tokens_seen": 49152000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.24904853940907}
\ No newline at end of file
diff --git a/checkpoints/metadata_000054034432.json b/checkpoints/metadata_000054034432.json
new file mode 100644
index 0000000000000000000000000000000000000000..392dee24269e4185647f6646640d79e803577eb4
--- /dev/null
+++ b/checkpoints/metadata_000054034432.json
@@ -0,0 +1 @@
+{"step": 3298, "tokens_seen": 54034432, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 4.111311179332413}
\ No newline at end of file
diff --git a/checkpoints/metadata_000057344000.json b/checkpoints/metadata_000057344000.json
new file mode 100644
index 0000000000000000000000000000000000000000..6295729bb77154914fc23f0af89f992e99ea64a7
--- /dev/null
+++ b/checkpoints/metadata_000057344000.json
@@ -0,0 +1 @@
+{"step": 3500, "tokens_seen": 57344000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.984464995565583}
\ No newline at end of file
diff --git a/checkpoints/metadata_000059441152.json b/checkpoints/metadata_000059441152.json
new file mode 100644
index 0000000000000000000000000000000000000000..b535540f48e281ae5a7052c1dd5968e54a890ac7
--- /dev/null
+++ b/checkpoints/metadata_000059441152.json
@@ -0,0 +1 @@
+{"step": 3628, "tokens_seen": 59441152, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.9296972093254294}
\ No newline at end of file
diff --git a/checkpoints/metadata_000065388544.json b/checkpoints/metadata_000065388544.json
new file mode 100644
index 0000000000000000000000000000000000000000..e43939c90853849b34a996784f3938f1b79be218
--- /dev/null
+++ b/checkpoints/metadata_000065388544.json
@@ -0,0 +1 @@
+{"step": 3991, "tokens_seen": 65388544, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.80356067287816}
\ No newline at end of file
diff --git a/checkpoints/metadata_000065536000.json b/checkpoints/metadata_000065536000.json
index 03f9872fed2e9157dd116e0dd8a80ff6f90f3c6a..fae4e50bc9ce9dab67b6646f8bb3bf9e599b9635 100644
--- a/checkpoints/metadata_000065536000.json
+++ b/checkpoints/metadata_000065536000.json
@@ -1 +1 @@
-{"step": 2000, "tokens_seen": 65536000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.892819235470663}
\ No newline at end of file
+{"step": 4000, "tokens_seen": 65536000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.805410613469591}
\ No newline at end of file
diff --git a/checkpoints/metadata_000071925760.json b/checkpoints/metadata_000071925760.json
new file mode 100644
index 0000000000000000000000000000000000000000..08823e05058e8856dda71b0c7232633af4a37c11
--- /dev/null
+++ b/checkpoints/metadata_000071925760.json
@@ -0,0 +1 @@
+{"step": 4390, "tokens_seen": 71925760, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.681084110888047}
\ No newline at end of file
diff --git a/checkpoints/metadata_000073728000.json b/checkpoints/metadata_000073728000.json
new file mode 100644
index 0000000000000000000000000000000000000000..67676aad23fbbc7ee102e47fd941a348325791c4
--- /dev/null
+++ b/checkpoints/metadata_000073728000.json
@@ -0,0 +1 @@
+{"step": 4500, "tokens_seen": 73728000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.64180357140983}
\ No newline at end of file
diff --git a/checkpoints/metadata_000079134720.json b/checkpoints/metadata_000079134720.json
new file mode 100644
index 0000000000000000000000000000000000000000..45a634ebbd82721a2373aa50a295474c5e67fb89
--- /dev/null
+++ b/checkpoints/metadata_000079134720.json
@@ -0,0 +1 @@
+{"step": 4830, "tokens_seen": 79134720, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.5811696154660817}
\ No newline at end of file
diff --git a/checkpoints/metadata_000081920000.json b/checkpoints/metadata_000081920000.json
index bce8604694de5b2f642941bcbcce0e4838c33b1c..cad9a7d74f8b17ce1ae1a2a6436c22cff2e42cb5 100644
--- a/checkpoints/metadata_000081920000.json
+++ b/checkpoints/metadata_000081920000.json
@@ -1 +1 @@
-{"step": 2500, "tokens_seen": 81920000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.6025823046975267}
\ No newline at end of file
+{"step": 5000, "tokens_seen": 81920000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.556630901397966}
\ No newline at end of file
diff --git a/checkpoints/metadata_000087048192.json b/checkpoints/metadata_000087048192.json
new file mode 100644
index 0000000000000000000000000000000000000000..0dad276fd310fbf00c37b3cf8a04ae4f08da189e
--- /dev/null
+++ b/checkpoints/metadata_000087048192.json
@@ -0,0 +1 @@
+{"step": 5313, "tokens_seen": 87048192, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.5201730770186788}
\ No newline at end of file
diff --git a/checkpoints/metadata_000090112000.json b/checkpoints/metadata_000090112000.json
new file mode 100644
index 0000000000000000000000000000000000000000..a0b5c60003ac756ccdca01193041df94037492e0
--- /dev/null
+++ b/checkpoints/metadata_000090112000.json
@@ -0,0 +1 @@
+{"step": 5500, "tokens_seen": 90112000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.4884309057190377}
\ No newline at end of file
diff --git a/checkpoints/metadata_000095748096.json b/checkpoints/metadata_000095748096.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e4d897edcc2c518a8530cc488edc8e09a0a721f
--- /dev/null
+++ b/checkpoints/metadata_000095748096.json
@@ -0,0 +1 @@
+{"step": 5844, "tokens_seen": 95748096, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.4558359774452776}
\ No newline at end of file
diff --git a/checkpoints/metadata_000098304000.json b/checkpoints/metadata_000098304000.json
index 52f8d0549b89044a9646cf6572937fafdc0797e2..0056bd2f5175921a34492a8581779895036ccd93 100644
--- a/checkpoints/metadata_000098304000.json
+++ b/checkpoints/metadata_000098304000.json
@@ -1 +1 @@
-{"step": 3000, "tokens_seen": 98304000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.468022187101252}
\ No newline at end of file
+{"step": 6000, "tokens_seen": 98304000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.448754568027084}
\ No newline at end of file
diff --git a/checkpoints/metadata_000105316352.json b/checkpoints/metadata_000105316352.json
new file mode 100644
index 0000000000000000000000000000000000000000..f164d241090cf3e4f4dd6e03b915146cc29b6692
--- /dev/null
+++ b/checkpoints/metadata_000105316352.json
@@ -0,0 +1 @@
+{"step": 6428, "tokens_seen": 105316352, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.3840863355982047}
\ No newline at end of file
diff --git a/checkpoints/metadata_000106496000.json b/checkpoints/metadata_000106496000.json
new file mode 100644
index 0000000000000000000000000000000000000000..6637822d53ba1e1229aa80aafd0fbb6d01193538
--- /dev/null
+++ b/checkpoints/metadata_000106496000.json
@@ -0,0 +1 @@
+{"step": 6500, "tokens_seen": 106496000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.3858616921553515}
\ No newline at end of file
diff --git a/checkpoints/metadata_000114688000.json b/checkpoints/metadata_000114688000.json
index 3fdab2eb102f9b8d6429a3046f93f4e4addb41ee..12fc826592c2e716014285f77cc2dc57f87ccd00 100644
--- a/checkpoints/metadata_000114688000.json
+++ b/checkpoints/metadata_000114688000.json
@@ -1 +1 @@
-{"step": 3500, "tokens_seen": 114688000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.3578212791343707}
\ No newline at end of file
+{"step": 7000, "tokens_seen": 114688000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.332638167603121}
\ No newline at end of file
diff --git a/checkpoints/metadata_000115851264.json b/checkpoints/metadata_000115851264.json
new file mode 100644
index 0000000000000000000000000000000000000000..506b666644c76fcffb32b326938ed4de0cb15e78
--- /dev/null
+++ b/checkpoints/metadata_000115851264.json
@@ -0,0 +1 @@
+{"step": 7071, "tokens_seen": 115851264, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.335663303226784}
\ No newline at end of file
diff --git a/checkpoints/metadata_000122880000.json b/checkpoints/metadata_000122880000.json
new file mode 100644
index 0000000000000000000000000000000000000000..a16fd2d7d947f49d0ee9b31230830606104ae3ad
--- /dev/null
+++ b/checkpoints/metadata_000122880000.json
@@ -0,0 +1 @@
+{"step": 7500, "tokens_seen": 122880000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.3251724437433485}
\ No newline at end of file
diff --git a/checkpoints/metadata_000127434752.json b/checkpoints/metadata_000127434752.json
new file mode 100644
index 0000000000000000000000000000000000000000..b8deb0e71c2165cffbf0688a1aa392d548e6c741
--- /dev/null
+++ b/checkpoints/metadata_000127434752.json
@@ -0,0 +1 @@
+{"step": 7778, "tokens_seen": 127434752, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.31138321475152}
\ No newline at end of file
diff --git a/checkpoints/metadata_000131072000.json b/checkpoints/metadata_000131072000.json
index 984c03867206a22b51058816392d69333c11e40e..aa3bbd65fdb0095554bf3137360076d85e799386 100644
--- a/checkpoints/metadata_000131072000.json
+++ b/checkpoints/metadata_000131072000.json
@@ -1 +1 @@
-{"step": 4000, "tokens_seen": 131072000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.3232916026020605}
\ No newline at end of file
+{"step": 8000, "tokens_seen": 131072000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.309278600746995}
\ No newline at end of file
diff --git a/checkpoints/metadata_000139264000.json b/checkpoints/metadata_000139264000.json
new file mode 100644
index 0000000000000000000000000000000000000000..dbaa86e333633bfffd2f7fd58f2c34ed441f1ccd
--- /dev/null
+++ b/checkpoints/metadata_000139264000.json
@@ -0,0 +1 @@
+{"step": 8500, "tokens_seen": 139264000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.2746704401860547}
\ No newline at end of file
diff --git a/checkpoints/metadata_000140181504.json b/checkpoints/metadata_000140181504.json
new file mode 100644
index 0000000000000000000000000000000000000000..5c9c4a195926b6cf8cb7aca5a022acfc53699adb
--- /dev/null
+++ b/checkpoints/metadata_000140181504.json
@@ -0,0 +1 @@
+{"step": 8556, "tokens_seen": 140181504, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.2705665368071903}
\ No newline at end of file
diff --git a/checkpoints/metadata_000147456000.json b/checkpoints/metadata_000147456000.json
index 919f6a3d70056e56609f9918ae5debfbf8729e7e..ae36f700a1aae921262a1b2c42ba31e9c2daaa6d 100644
--- a/checkpoints/metadata_000147456000.json
+++ b/checkpoints/metadata_000147456000.json
@@ -1 +1 @@
-{"step": 4500, "tokens_seen": 147456000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.2607066672599236}
\ No newline at end of file
+{"step": 9000, "tokens_seen": 147456000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.234525672614021}
\ No newline at end of file
diff --git a/checkpoints/metadata_000154206208.json b/checkpoints/metadata_000154206208.json
new file mode 100644
index 0000000000000000000000000000000000000000..c54f5c53a44129ffc47c237a653ae90149015df6
--- /dev/null
+++ b/checkpoints/metadata_000154206208.json
@@ -0,0 +1 @@
+{"step": 9412, "tokens_seen": 154206208, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.2300571386274206}
\ No newline at end of file
diff --git a/checkpoints/metadata_000155648000.json b/checkpoints/metadata_000155648000.json
new file mode 100644
index 0000000000000000000000000000000000000000..e65af06ecc65ab88400125cc0e9508d2a1ee1e97
--- /dev/null
+++ b/checkpoints/metadata_000155648000.json
@@ -0,0 +1 @@
+{"step": 9500, "tokens_seen": 155648000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.228658600438954}
\ No newline at end of file
diff --git a/checkpoints/metadata_000163840000.json b/checkpoints/metadata_000163840000.json
index 7c93c82aaec4b9e1a3ac7ef6d4849eb5b689c16c..846316fa8560931e4e606fedbfb982b2dd8537a4 100644
--- a/checkpoints/metadata_000163840000.json
+++ b/checkpoints/metadata_000163840000.json
@@ -1 +1 @@
-{"step": 5000, "tokens_seen": 163840000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.219682038509445}
\ No newline at end of file
+{"step": 10000, "tokens_seen": 163840000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.199526206936364}
\ No newline at end of file
diff --git a/checkpoints/metadata_000169623552.json b/checkpoints/metadata_000169623552.json
new file mode 100644
index 0000000000000000000000000000000000000000..403b21bd2c22d7691afa458f2c6f4dd15a5eab75
--- /dev/null
+++ b/checkpoints/metadata_000169623552.json
@@ -0,0 +1 @@
+{"step": 10353, "tokens_seen": 169623552, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.2004114891693147}
\ No newline at end of file
diff --git a/checkpoints/metadata_000172032000.json b/checkpoints/metadata_000172032000.json
new file mode 100644
index 0000000000000000000000000000000000000000..77a3272a6782e448b87ebc76d1a9c546d5ec24c8
--- /dev/null
+++ b/checkpoints/metadata_000172032000.json
@@ -0,0 +1 @@
+{"step": 10500, "tokens_seen": 172032000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.2056609005062953}
\ No newline at end of file
diff --git a/checkpoints/metadata_000180224000.json b/checkpoints/metadata_000180224000.json
index 6ae9979b89dcaeb66bfea8204564188e5859f022..b2916f82e5dcc3f6fc98307c8e77a71dccdedfcb 100644
--- a/checkpoints/metadata_000180224000.json
+++ b/checkpoints/metadata_000180224000.json
@@ -1 +1 @@
-{"step": 5500, "tokens_seen": 180224000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.186685546578624}
\ No newline at end of file
+{"step": 11000, "tokens_seen": 180224000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1686492056920414}
\ No newline at end of file
diff --git a/checkpoints/metadata_000186580992.json b/checkpoints/metadata_000186580992.json
new file mode 100644
index 0000000000000000000000000000000000000000..87c2b50a39418cdc546cb4d585611d4d67c81ac1
--- /dev/null
+++ b/checkpoints/metadata_000186580992.json
@@ -0,0 +1 @@
+{"step": 11388, "tokens_seen": 186580992, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.153817280911148}
\ No newline at end of file
diff --git a/checkpoints/metadata_000188416000.json b/checkpoints/metadata_000188416000.json
new file mode 100644
index 0000000000000000000000000000000000000000..d535c11a3d6c0421ce0fb309ba970afa2aa0397e
--- /dev/null
+++ b/checkpoints/metadata_000188416000.json
@@ -0,0 +1 @@
+{"step": 11500, "tokens_seen": 188416000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.161236533029193}
\ No newline at end of file
diff --git a/checkpoints/metadata_000196608000.json b/checkpoints/metadata_000196608000.json
index be9927f83f64923f3e9863bedac7bdc20c9fc049..723ca80bda508b65d1a5a9ba54d02aa5b4ff38dc 100644
--- a/checkpoints/metadata_000196608000.json
+++ b/checkpoints/metadata_000196608000.json
@@ -1 +1 @@
-{"step": 6000, "tokens_seen": 196608000, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.1691659182780376}
\ No newline at end of file
+{"step": 12000, "tokens_seen": 196608000, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.138256716618341}
\ No newline at end of file
diff --git a/checkpoints/metadata_000198361088.json b/checkpoints/metadata_000198361088.json
new file mode 100644
index 0000000000000000000000000000000000000000..3a4317b61e5a7fd27fcd92a8a88be7c7e5c0bfcb
--- /dev/null
+++ b/checkpoints/metadata_000198361088.json
@@ -0,0 +1 @@
+{"step": 12107, "tokens_seen": 198361088, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1362967564356645}
\ No newline at end of file
diff --git a/checkpoints/metadata_000198688768.json b/checkpoints/metadata_000198688768.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a407101b50b795e04313841c909a6a9cac3714b
--- /dev/null
+++ b/checkpoints/metadata_000198688768.json
@@ -0,0 +1 @@
+{"step": 12127, "tokens_seen": 198688768, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.127549854661895}
\ No newline at end of file
diff --git a/checkpoints/metadata_000199016448.json b/checkpoints/metadata_000199016448.json
new file mode 100644
index 0000000000000000000000000000000000000000..14164fdc8fd4e51677e3e11a2eb8e93d82e3c0bd
--- /dev/null
+++ b/checkpoints/metadata_000199016448.json
@@ -0,0 +1 @@
+{"step": 12147, "tokens_seen": 199016448, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.132057432713208}
\ No newline at end of file
diff --git a/checkpoints/metadata_000199344128.json b/checkpoints/metadata_000199344128.json
new file mode 100644
index 0000000000000000000000000000000000000000..a5d1e28a2b6e0b7d31abf6fb061ef250e217f6ea
--- /dev/null
+++ b/checkpoints/metadata_000199344128.json
@@ -0,0 +1 @@
+{"step": 12167, "tokens_seen": 199344128, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.139671201985256}
\ No newline at end of file
diff --git a/checkpoints/metadata_000199671808.json b/checkpoints/metadata_000199671808.json
new file mode 100644
index 0000000000000000000000000000000000000000..e55f54c9edd9fa8bf9404378df28c160f948e7ae
--- /dev/null
+++ b/checkpoints/metadata_000199671808.json
@@ -0,0 +1 @@
+{"step": 12187, "tokens_seen": 199671808, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1339797231647304}
\ No newline at end of file
diff --git a/checkpoints/metadata_000199983104.json b/checkpoints/metadata_000199983104.json
new file mode 100644
index 0000000000000000000000000000000000000000..b89a001b93d5048c0cb80071284c2e6c56a8fc4e
--- /dev/null
+++ b/checkpoints/metadata_000199983104.json
@@ -0,0 +1 @@
+{"step": 12206, "tokens_seen": 199983104, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1248199771775043}
\ No newline at end of file
diff --git a/checkpoints/model_weights_000000016384.pt b/checkpoints/model_weights_000000016384.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2629b5772c9dd25495de464d094e74ed851ed9d5
--- /dev/null
+++ b/checkpoints/model_weights_000000016384.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d67a6703b7fce680b012ef63412b92220d3c4bb610cdf19d2616be10e6ff805
+size 284687203
diff --git a/checkpoints/model_weights_000000163840.pt b/checkpoints/model_weights_000000163840.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14a956def213ef911107f553b6ccf02eb267345a
--- /dev/null
+++ b/checkpoints/model_weights_000000163840.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eba92b94b7939bdd744a02dd1aa6797cc133a32a59500f9572f3a96d4470a63
+size 284687203
diff --git a/checkpoints/model_weights_000000180224.pt b/checkpoints/model_weights_000000180224.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25308c8f044369ba3e7dd183ebbae5e6903c12df
--- /dev/null
+++ b/checkpoints/model_weights_000000180224.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a235ca8e89fa7240ba5d85255349bd676ffb52dfb4812c5a072e0d1b509851
+size 284687203
diff --git a/checkpoints/model_weights_000000212992.pt b/checkpoints/model_weights_000000212992.pt
new file mode 100644
index 0000000000000000000000000000000000000000..520b1b940e7669ba222a0968ea6fd6f0c72593c2
--- /dev/null
+++ b/checkpoints/model_weights_000000212992.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7558d5b08ac93c36974aa0c6a37e27712029f4bea26932bb9ca934a707fdf0b
+size 284687203
diff --git a/checkpoints/model_weights_000000229376.pt b/checkpoints/model_weights_000000229376.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fe04a6ac2314c0d59ea61ae89ee6900fc600116
--- /dev/null
+++ b/checkpoints/model_weights_000000229376.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddb79be005c02494c70d6795d59c92509f9ed09e256c69be40fe08775644d8a4
+size 284687203
diff --git a/checkpoints/model_weights_000000245760.pt b/checkpoints/model_weights_000000245760.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4bbb79cce36979bdf569ef5ea93013db1f8214e3
--- /dev/null
+++ b/checkpoints/model_weights_000000245760.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fade4b7a47acf8eabf7da25911a5b7b2dec742ccbf183b786340c4085e154322
+size 284687203
diff --git a/checkpoints/model_weights_000000278528.pt b/checkpoints/model_weights_000000278528.pt
new file mode 100644
index 0000000000000000000000000000000000000000..356cad1cd41fa0caf575bcdff56cff12ddf5a350
--- /dev/null
+++ b/checkpoints/model_weights_000000278528.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488862fb730839e4a728e7ba97ff7599345cd419dca0b86e54d7fa89fc810946
+size 284687203
diff --git a/checkpoints/model_weights_000000311296.pt b/checkpoints/model_weights_000000311296.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfe150ffa1d01684f46534b2c897fe06add4c649
--- /dev/null
+++ b/checkpoints/model_weights_000000311296.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcfd431fbf205b6dbbc144d8a4927f5025abe2f5c433599810d974f56d97afbf
+size 284687203
diff --git a/checkpoints/model_weights_000000344064.pt b/checkpoints/model_weights_000000344064.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c05bcb7424298ac1a9d94c1638e570599a673f31
--- /dev/null
+++ b/checkpoints/model_weights_000000344064.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccd4a6e0e6309422df352f92a09339a67867b1150f2385724de828dc9f217580
+size 284687203
diff --git a/checkpoints/model_weights_000000376832.pt b/checkpoints/model_weights_000000376832.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a1c7acb3fd5094c3ab0b0cc812be65be6dbea422
--- /dev/null
+++ b/checkpoints/model_weights_000000376832.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad35f6d6610ee1618040d2a5b1d51c3d506384b093b93420d18b50a055baa6ef
+size 284687203
diff --git a/checkpoints/model_weights_000000409600.pt b/checkpoints/model_weights_000000409600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f659ea5b007d058ea1e7ae8fba50e166e6116a8f
--- /dev/null
+++ b/checkpoints/model_weights_000000409600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a279a1242f021b968b9f38c5250eebc55c3bf17d2dccf1a2cf52a5fe96abb7
+size 284687203
diff --git a/checkpoints/model_weights_000000458752.pt b/checkpoints/model_weights_000000458752.pt
index 5134c42f22ce4bfaa26b5baeae27b8e9b9ec15be..746bd3ccaf3d69b43ee3c2be6a2fc1855571a63e 100644
--- a/checkpoints/model_weights_000000458752.pt
+++ b/checkpoints/model_weights_000000458752.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ed191da20dfbb2e1ab5ec35861168f2b323a668d5e12e87fba7481cd1c8a6a3
-size 234226143
+oid sha256:14c21a764eef3ea325226ec63f9f33dae1df0ef038a0a14d40fc8227ad168479
+size 284687203
diff --git a/checkpoints/model_weights_000000491520.pt b/checkpoints/model_weights_000000491520.pt
index 03b775da958ab4f5f58213445fa71e86dbe41d6c..c9fcda64d5a9be27a93ccb8a40489c57bfe10880 100644
--- a/checkpoints/model_weights_000000491520.pt
+++ b/checkpoints/model_weights_000000491520.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b1ec2fc29759ec5487e365bef4114629736e4ff29a8be4e8b307eecdd9783d7
-size 234226143
+oid sha256:0f3b5321be86eefe9a862a3cb2f5ead22656f3a1160424c20bdb59e4808404e5
+size 284687203
diff --git a/checkpoints/model_weights_000000557056.pt b/checkpoints/model_weights_000000557056.pt
index 0d77ae0210a65259418e216445b195e14a0e2cb9..7b6bfe1a5940ae983e3a4a5cbfaa3af89cefa1d0 100644
--- a/checkpoints/model_weights_000000557056.pt
+++ b/checkpoints/model_weights_000000557056.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4dc3b57140362171dd1aa9a0019cae21ae325876a30580ccfa61d73895273ba1
-size 234226143
+oid sha256:41cd7c2367161f15614475bddf6bcf5b766ee7629b48d634017c1c7e6bd38674
+size 284687203
diff --git a/checkpoints/model_weights_000000606208.pt b/checkpoints/model_weights_000000606208.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4b16d55991f1fb6cc4c6650c157b6c6c76a2c40
--- /dev/null
+++ b/checkpoints/model_weights_000000606208.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f87569a6062576686034f5acaeae3d2ae8fd23d16db94d94e3006ed841ae44f2
+size 284687203
diff --git a/checkpoints/model_weights_000000671744.pt b/checkpoints/model_weights_000000671744.pt
new file mode 100644
index 0000000000000000000000000000000000000000..259e69167d8f1fcc28756b6f99738820fb40efb2
--- /dev/null
+++ b/checkpoints/model_weights_000000671744.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d820e53cc47c309efc44c8f0d23f03bab9a83039eda311f2f14ebdd4ed3fb9c
+size 284687203
diff --git a/checkpoints/model_weights_000000737280.pt b/checkpoints/model_weights_000000737280.pt
new file mode 100644
index 0000000000000000000000000000000000000000..705b1a950527a5793fce8a9634f4e5cf41176234
--- /dev/null
+++ b/checkpoints/model_weights_000000737280.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf7ed1a09c9ac8ce4bf42c5b700dcb4e0400ee0fe6a0443b83e2e799a467ad12
+size 284687203
diff --git a/checkpoints/model_weights_000000802816.pt b/checkpoints/model_weights_000000802816.pt
new file mode 100644
index 0000000000000000000000000000000000000000..904f066f0cb91132d3caf851549242194f533a12
--- /dev/null
+++ b/checkpoints/model_weights_000000802816.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eca171cdecadf26058cdc6562906a2c48610b0562c35d3527883d7b0f38af4ce
+size 284687203
diff --git a/checkpoints/model_weights_000000884736.pt b/checkpoints/model_weights_000000884736.pt
new file mode 100644
index 0000000000000000000000000000000000000000..397edf7ef4ae8efc65372e1e6db650cc7b32a02e
--- /dev/null
+++ b/checkpoints/model_weights_000000884736.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c41ab9e097b621476a9efa33c7c54d29ca4d9f9c349940d43d51499986447be
+size 284687203
diff --git a/checkpoints/model_weights_000000983040.pt b/checkpoints/model_weights_000000983040.pt
index 80d79333576ccc2bc86a2bbad3c288736acdfcc0..d4747a7df432b00dc21d3d4ad11ef5b8995f0c74 100644
--- a/checkpoints/model_weights_000000983040.pt
+++ b/checkpoints/model_weights_000000983040.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fcc3047289920770967eef0c6c14741f74165f6da09765b641c50297f6e0814
-size 234226143
+oid sha256:44c3ce79770dd2a3bd2b78484b0259178f03d92c7c3227f32927a22160b0a665
+size 284687203
diff --git a/checkpoints/model_weights_000001081344.pt b/checkpoints/model_weights_000001081344.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c275db09b146def1fa3c83d2d364b81726c9c7e9
--- /dev/null
+++ b/checkpoints/model_weights_000001081344.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2f19c4205f78ebffad6efe1b689150f4a33f630c30384859a761d573d33a6d6
+size 284687203
diff --git a/checkpoints/model_weights_000001179648.pt b/checkpoints/model_weights_000001179648.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d542da349672a5fb5de470f1fa993ec2b52d9c02
--- /dev/null
+++ b/checkpoints/model_weights_000001179648.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45d0cea1bb24bd0f5e900aecc9e3b74371e1339d8859151c7fc38463dbf4f3e3
+size 284687203
diff --git a/checkpoints/model_weights_000001310720.pt b/checkpoints/model_weights_000001310720.pt
new file mode 100644
index 0000000000000000000000000000000000000000..067ce96137c5fa51f53d3515e6cce20359ec24c7
--- /dev/null
+++ b/checkpoints/model_weights_000001310720.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08808d6a961a77adad34359b753210b25e553eabb6f51a64b6682014dd9bdbcb
+size 284687203
diff --git a/checkpoints/model_weights_000001441792.pt b/checkpoints/model_weights_000001441792.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a50018b409d1b123726547d3cfbfcde24b56076
--- /dev/null
+++ b/checkpoints/model_weights_000001441792.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c192cfc67f1f103661f648d285b7a6f7f5332b2f9c53657dd789d747ec9945a
+size 284687203
diff --git a/checkpoints/model_weights_000001589248.pt b/checkpoints/model_weights_000001589248.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ddcccef46f9638599f624fa3684f8e05b59b5df1
--- /dev/null
+++ b/checkpoints/model_weights_000001589248.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:522220d513ed5630e21cc5af61c776ff29ae6304ed495b27fa14059aee75d0e0
+size 284687203
diff --git a/checkpoints/model_weights_000001736704.pt b/checkpoints/model_weights_000001736704.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1f6d574c86d6b92935ea397873fe318b452234a
--- /dev/null
+++ b/checkpoints/model_weights_000001736704.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b23fabb9becc887ae3129605126d27e675eaa6b44ff1025b3186f41cc7b86189
+size 284687203
diff --git a/checkpoints/model_weights_000001916928.pt b/checkpoints/model_weights_000001916928.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1ed4557fb9e74246c18e9758f028b2c8bc51ca08
--- /dev/null
+++ b/checkpoints/model_weights_000001916928.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d4bdcad787ac9a051115abad63ef658c0300bac8980321cf20234156ab71a1f
+size 284687203
diff --git a/checkpoints/model_weights_000002113536.pt b/checkpoints/model_weights_000002113536.pt
new file mode 100644
index 0000000000000000000000000000000000000000..247e65b251d2121d64379d2520844c1dd90d6b5d
--- /dev/null
+++ b/checkpoints/model_weights_000002113536.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f40e360e7e9093b3cdf500bf6af1e3836a119a99fa09fa18c83874a8ffd02edd
+size 284687203
diff --git a/checkpoints/model_weights_000002326528.pt b/checkpoints/model_weights_000002326528.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba313f84d2034eebd931de5428562ecd15c77167
--- /dev/null
+++ b/checkpoints/model_weights_000002326528.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d50e7f5dd748336b0bac67a29171f80b022fca2f99529065aa3ae5aa8cbafcb
+size 284687203
diff --git a/checkpoints/model_weights_000002555904.pt b/checkpoints/model_weights_000002555904.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f60daa37870b6cc41634ffea0a30b9534fd38a45
--- /dev/null
+++ b/checkpoints/model_weights_000002555904.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fead2a6fb8d77a849252864b5ac5decb6cfb0c209a8b8edf2bc9a8da0c05cbd2
+size 284687203
diff --git a/checkpoints/model_weights_000002801664.pt b/checkpoints/model_weights_000002801664.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2bb9091b915ee52eb9275e69b6e4a0c42596a567
--- /dev/null
+++ b/checkpoints/model_weights_000002801664.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b474e31475941a3bdcf83d19d931e05aac3189c23d91bdb7a863d355d2d115
+size 284687203
diff --git a/checkpoints/model_weights_000003096576.pt b/checkpoints/model_weights_000003096576.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4826cc49807ede3d946826653c1715bd39793d4
--- /dev/null
+++ b/checkpoints/model_weights_000003096576.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01ff966235c4e372b840d3c1d37d5f873a547e409d80a550de85971c33ae2f4a
+size 284687203
diff --git a/checkpoints/model_weights_000003391488.pt b/checkpoints/model_weights_000003391488.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c40b316efc8d99e4de8773c643c01907b6f6f8d5
--- /dev/null
+++ b/checkpoints/model_weights_000003391488.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e8a987a6895c953d161838f05a0c05e165f6724adc4eeffcee8e1cb014f97d
+size 284687203
diff --git a/checkpoints/model_weights_000003735552.pt b/checkpoints/model_weights_000003735552.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd9e8b15108fa4b754a083b9219c8d824c3bfe34
--- /dev/null
+++ b/checkpoints/model_weights_000003735552.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12403085a7d626a2681ffcd80555417e0d71830f928612e3986a2e67aab8e9cc
+size 284687203
diff --git a/checkpoints/model_weights_000004112384.pt b/checkpoints/model_weights_000004112384.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32eb2e2c54c7029906b45bf6705ee247a417b529
--- /dev/null
+++ b/checkpoints/model_weights_000004112384.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27d2b6865bcd758f2a65e5a9a440907fe84e714b305c059cb739a1c10a41cd1
+size 284687203
diff --git a/checkpoints/model_weights_000004521984.pt b/checkpoints/model_weights_000004521984.pt
new file mode 100644
index 0000000000000000000000000000000000000000..410c2480ecb901d99ebcada10dc2089b0429ffa8
--- /dev/null
+++ b/checkpoints/model_weights_000004521984.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e378c5df35246ff226d3d2be7a9956bbe196ebdfe3aa4dc7232de3b4a9dd4163
+size 284687203
diff --git a/checkpoints/model_weights_000004980736.pt b/checkpoints/model_weights_000004980736.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f42bd95996df7eb990fd06aed8e8440b3f0c09c5
--- /dev/null
+++ b/checkpoints/model_weights_000004980736.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05f078068469fc47d30ed53d73f5cc2ca92360b7cf9689eed166fbadab14783d
+size 284687203
diff --git a/checkpoints/model_weights_000005472256.pt b/checkpoints/model_weights_000005472256.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78fa54c11f824f80a722facd871d5e2ed8777dc6
--- /dev/null
+++ b/checkpoints/model_weights_000005472256.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24500835388f5cfe24beac1adf69e0d214ca155a874a8e2af15953ad34342ee9
+size 284687203
diff --git a/checkpoints/model_weights_000006029312.pt b/checkpoints/model_weights_000006029312.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63299bfd4ef00c064f5e7c089210fada80678ed7
--- /dev/null
+++ b/checkpoints/model_weights_000006029312.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17a12e39cacbde578ade186214656d3a1b95b08f31d4e0aa614ac4a12a80457a
+size 284687203
diff --git a/checkpoints/model_weights_000006635520.pt b/checkpoints/model_weights_000006635520.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c05cabe242536071b2394610610e6a55266cd6a
--- /dev/null
+++ b/checkpoints/model_weights_000006635520.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a6e510f35a1ce7446ef8c5feb5c23c3d064578a93ab99f7e1d30c1a2b22edb6
+size 284687203
diff --git a/checkpoints/model_weights_000007290880.pt b/checkpoints/model_weights_000007290880.pt
new file mode 100644
index 0000000000000000000000000000000000000000..869815fa8fa1b7815735cb25fb772b315e90b116
--- /dev/null
+++ b/checkpoints/model_weights_000007290880.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1628c8fc113767328f79954296872b31f68764d5702d6599415e8582241504f
+size 284687203
diff --git a/checkpoints/model_weights_000008028160.pt b/checkpoints/model_weights_000008028160.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1555f554202c19478f4ac887d03b25bf785059b4
--- /dev/null
+++ b/checkpoints/model_weights_000008028160.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b526e476cfe8eb2b180ee3fb5e06ef8435a78225f8bcb0227d0c029d3f3be25
+size 284687203
diff --git a/checkpoints/model_weights_000008192000.pt b/checkpoints/model_weights_000008192000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fda6a2823550130e3b2528cd06d0cb5783245dc
--- /dev/null
+++ b/checkpoints/model_weights_000008192000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c5d860bae18aa5868608155eb75c5f4d2dd324f77cee271d0c8e5fe08659d2a
+size 284687203
diff --git a/checkpoints/model_weights_000008830976.pt b/checkpoints/model_weights_000008830976.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e78a0859db98608b16c1899e8b0eb40939bbbc1
--- /dev/null
+++ b/checkpoints/model_weights_000008830976.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49071a63070fa14918458a6f9aca95e24cefe45b179ac2170203224875c69c36
+size 284687203
diff --git a/checkpoints/model_weights_000009715712.pt b/checkpoints/model_weights_000009715712.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb1abbbf3d4e4fd57273ec5e37ebf251955aa03c
--- /dev/null
+++ b/checkpoints/model_weights_000009715712.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:404f46b30bcd371279361cd37fed56c569a5e2da3d0122a41affc44e261bb960
+size 284687203
diff --git a/checkpoints/model_weights_000010682368.pt b/checkpoints/model_weights_000010682368.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bfe3a9da3d144d34edd792fb54cfcac020a3d280
--- /dev/null
+++ b/checkpoints/model_weights_000010682368.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd1788b2b576df081e9d5a1cc3ee5874d052729b44367a40975a1fed033636c9
+size 284687203
diff --git a/checkpoints/model_weights_000011747328.pt b/checkpoints/model_weights_000011747328.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94f60228c34d2f7917a90eb6d918334eaccc454f
--- /dev/null
+++ b/checkpoints/model_weights_000011747328.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10012b9b93201125b17becc26ae57492214d32ffeea6fcd029e40ce07b7b8e2d
+size 284687203
diff --git a/checkpoints/model_weights_000012926976.pt b/checkpoints/model_weights_000012926976.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64b314b43dda41c390b8d2aa24befca08adb7005
--- /dev/null
+++ b/checkpoints/model_weights_000012926976.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c835380e3a341efee6271f085095cdf8a47e0efd5360abfe14eed94ec3f095ee
+size 284687203
diff --git a/checkpoints/model_weights_000014221312.pt b/checkpoints/model_weights_000014221312.pt
new file mode 100644
index 0000000000000000000000000000000000000000..642452646ec4131cbaef4e582d9745785e44de7e
--- /dev/null
+++ b/checkpoints/model_weights_000014221312.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06383b9a9f33c82114e235e9f1bddd51b1554141fb63dbb059a94c667bb3944c
+size 284687203
diff --git a/checkpoints/model_weights_000015646720.pt b/checkpoints/model_weights_000015646720.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da18e5615872f7d45b0834939904aa5aad0039e6
--- /dev/null
+++ b/checkpoints/model_weights_000015646720.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:452d3ae6451715c659034d84c7f4c6f0369179da80d908975a8cf60623ae71a8
+size 284687203
diff --git a/checkpoints/model_weights_000016384000.pt b/checkpoints/model_weights_000016384000.pt
index 735ad8aaf85522fe1073f724336a222df762b8dd..88c5b13117f9b7f9f31fb0a4ac3c62f2cfc6e57f 100644
--- a/checkpoints/model_weights_000016384000.pt
+++ b/checkpoints/model_weights_000016384000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dee7ce0f45dd723646197a9f196cfad58d74ff5540a6dd950743d4ea0fd08012
-size 234226143
+oid sha256:a971fa39b77f0e1501d61343e528b5986733d09f30cf9b5ee9b7570588e617a7
+size 284687203
diff --git a/checkpoints/model_weights_000017219584.pt b/checkpoints/model_weights_000017219584.pt
new file mode 100644
index 0000000000000000000000000000000000000000..830785c47555c9bf416bac7b398d698cbcab9497
--- /dev/null
+++ b/checkpoints/model_weights_000017219584.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6daffaee80a7842f1277abfe55da2dcabd20c0d714e370ab697c79bb15dbbda9
+size 284687203
diff --git a/checkpoints/model_weights_000018939904.pt b/checkpoints/model_weights_000018939904.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2823026d885f384bedad2aab8a9d94aaa7c66fc0
--- /dev/null
+++ b/checkpoints/model_weights_000018939904.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4655e3f65c84207dd341bd6113198c03288228bcbd881d4055416cb48e3725bb
+size 284687203
diff --git a/checkpoints/model_weights_000020824064.pt b/checkpoints/model_weights_000020824064.pt
new file mode 100644
index 0000000000000000000000000000000000000000..32bd2755ae27444362222f88380328aaac23d2d2
--- /dev/null
+++ b/checkpoints/model_weights_000020824064.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eec1a09e3705b5043d65ff82e1e807e0525e8536f36b112ef9928b46ab6415b
+size 284687203
diff --git a/checkpoints/model_weights_000022921216.pt b/checkpoints/model_weights_000022921216.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6a4c4ece0833cbd5c403b6991dec21b27d77cf9
--- /dev/null
+++ b/checkpoints/model_weights_000022921216.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce059b79c518e5f3552fc93bcc2d00b548e49f9cf336fd05a3678e1e82090154
+size 284687203
diff --git a/checkpoints/model_weights_000024576000.pt b/checkpoints/model_weights_000024576000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a3cbb18b8086f2ee2e198f40ba5c36fa6d0ba71
--- /dev/null
+++ b/checkpoints/model_weights_000024576000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b44932e4c672a4da9c808426054ece74b995bb5a9cf0d507283fb1741e8e086b
+size 284687203
diff --git a/checkpoints/model_weights_000025198592.pt b/checkpoints/model_weights_000025198592.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe072d8c0aa29a8dd6fc6205744a49b5e76132d2
--- /dev/null
+++ b/checkpoints/model_weights_000025198592.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66bfdca3db900cf83362edae2f0da91cf2680c0b3034fad6656643f7b2f588c0
+size 284687203
diff --git a/checkpoints/model_weights_000027721728.pt b/checkpoints/model_weights_000027721728.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c93e69874071b74a9ba013a7842ee874b7ca22a
--- /dev/null
+++ b/checkpoints/model_weights_000027721728.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67bc9fa9bb5be827346711c54dc2e18aaf90ffcf6842bc241bc50343a65f8883
+size 284687203
diff --git a/checkpoints/model_weights_000030507008.pt b/checkpoints/model_weights_000030507008.pt
new file mode 100644
index 0000000000000000000000000000000000000000..426a0e3f4ed0e7970354528a4549b00006981d21
--- /dev/null
+++ b/checkpoints/model_weights_000030507008.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e1eddd7343280e644d1c089f5be7441f948d1d174a2057e45166bee13140b2
+size 284687203
diff --git a/checkpoints/model_weights_000032768000.pt b/checkpoints/model_weights_000032768000.pt
index 7ba84ead13efa990f176de34d228ea97677eb226..311606a121e0e6fee14d373b1e40ea8487926035 100644
--- a/checkpoints/model_weights_000032768000.pt
+++ b/checkpoints/model_weights_000032768000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4caac26e32c45f3785dd5020d02dd1a7161860a82614dc7292138564d6030c93
-size 234226143
+oid sha256:6e37512e6a3522edc964c7ad6e99b681ae41caf8fb0266db04a0cc7a50157456
+size 284687203
diff --git a/checkpoints/model_weights_000033554432.pt b/checkpoints/model_weights_000033554432.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b081889d72e6a17416830bd05a092d543fb339a7
--- /dev/null
+++ b/checkpoints/model_weights_000033554432.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59aa5fe8e7b3b4705509b97af80bc3efe1f7a47fde9d22f483cf311f43a4eec4
+size 284687203
diff --git a/checkpoints/model_weights_000036913152.pt b/checkpoints/model_weights_000036913152.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7893bb87978e7a5e434caa3d3879aac9ff273515
--- /dev/null
+++ b/checkpoints/model_weights_000036913152.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4535aa23f9e484791c5d3dc8536f492bd29abf7a976187e74d928563a62503b
+size 284687203
diff --git a/checkpoints/model_weights_000040599552.pt b/checkpoints/model_weights_000040599552.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3ffd40c08264875669a1913d8a03777f34c0801
--- /dev/null
+++ b/checkpoints/model_weights_000040599552.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:830a396e7e218ec5333677744f33061b7b3e5168224c17e4784c736c76afa9f4
+size 284687203
diff --git a/checkpoints/model_weights_000040960000.pt b/checkpoints/model_weights_000040960000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2c157ae702a300589f6a5770aad4aaa5e121bde4
--- /dev/null
+++ b/checkpoints/model_weights_000040960000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ffec59ce4964faec9f70abc759e74b7c6ea381cad5fa4b4e262d809b8640273
+size 284687203
diff --git a/checkpoints/model_weights_000044662784.pt b/checkpoints/model_weights_000044662784.pt
new file mode 100644
index 0000000000000000000000000000000000000000..098f1b1cc817dec31e0e425baad8e86d6e2ccdd7
--- /dev/null
+++ b/checkpoints/model_weights_000044662784.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d15c4ae6f0b97f264376de1f8053ba3cc4cc705a89610d48e64d370021f06600
+size 284687203
diff --git a/checkpoints/model_weights_000049135616.pt b/checkpoints/model_weights_000049135616.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aaea4e1763fce84470bbc7a8d46164da9bbb46f9
--- /dev/null
+++ b/checkpoints/model_weights_000049135616.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:237cefffdeacaf241858235faaeea4e6a571abf523c8c3741848158c10a93a0b
+size 284687203
diff --git a/checkpoints/model_weights_000049152000.pt b/checkpoints/model_weights_000049152000.pt
index b4b2e0f91f29e800f9506d014d4c9b66b8a19fa2..d0be8a2077f9d73023097d479c58944aab678e13 100644
--- a/checkpoints/model_weights_000049152000.pt
+++ b/checkpoints/model_weights_000049152000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5b4dd998b47d1fb49d6e2cc31df3988a300a8c14a7b0e148687dd5227370a2d
-size 234226143
+oid sha256:184d0f82bc99a936ab4bbb8440cc8270c12f5196b72c3be323f74a17b73ec5ab
+size 284687203
diff --git a/checkpoints/model_weights_000054034432.pt b/checkpoints/model_weights_000054034432.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82527a320d0adffe4c66f51ea129eaab9c67374f
--- /dev/null
+++ b/checkpoints/model_weights_000054034432.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2df933b6cf887b1993147a73ac1b688a14d2494a57876f379b3f6bfb7ddf267
+size 284687203
diff --git a/checkpoints/model_weights_000057344000.pt b/checkpoints/model_weights_000057344000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..49d0d101cd3accd61be10a736a3d4f2d84c2bc93
--- /dev/null
+++ b/checkpoints/model_weights_000057344000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2ff574ec6ff6b04fd691faf26e97fba601f47e699f64bd1d4a5f1ae045c3c6a
+size 284687203
diff --git a/checkpoints/model_weights_000059441152.pt b/checkpoints/model_weights_000059441152.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0aa931e0c2f5a0aefcf2fa8d6f96b6b9944d39a4
--- /dev/null
+++ b/checkpoints/model_weights_000059441152.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bd808126637845599312780a2ee6ed8b49498cdb8b06820709d8566d5d47c74
+size 284687203
diff --git a/checkpoints/model_weights_000065388544.pt b/checkpoints/model_weights_000065388544.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea159d9f8105ba74d06f55dc0db12f7536c8946c
--- /dev/null
+++ b/checkpoints/model_weights_000065388544.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e94d1b3a0fa7f6e650561d43d3c577fcce96ea6fed69c04aec585c1d50ecad72
+size 284687203
diff --git a/checkpoints/model_weights_000065536000.pt b/checkpoints/model_weights_000065536000.pt
index 578588dd1bd7a0f634a00c918487ccfb78e6bd68..cd897e5707ea606db332329a6d77b268f6d9cb53 100644
--- a/checkpoints/model_weights_000065536000.pt
+++ b/checkpoints/model_weights_000065536000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92e572ed93ae54bdf02f9099dc87b87ce1b810b63e71e3ed67e76cc7d08659ff
-size 234226143
+oid sha256:6bf57cd0c89b53c0ab805f25e16bac1ded3d6aee1f901b52a07969a79ac70cd6
+size 284687203
diff --git a/checkpoints/model_weights_000071925760.pt b/checkpoints/model_weights_000071925760.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ee960c4a31ec3af9671cbd7bbfb25e5bc30366e
--- /dev/null
+++ b/checkpoints/model_weights_000071925760.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ddeb04c5377f286498c6a2658b8df752b9279d73fbcd06990eacfb33fa1e6e3
+size 284687203
diff --git a/checkpoints/model_weights_000073728000.pt b/checkpoints/model_weights_000073728000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..017045f302d107d1a53e5e03be2c76cc7bc1497a
--- /dev/null
+++ b/checkpoints/model_weights_000073728000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b1c4c2d3b857be7b03d849eeda94b3e6ecbfbcb154cdd021f52aa680a0ee0a
+size 284687203
diff --git a/checkpoints/model_weights_000079134720.pt b/checkpoints/model_weights_000079134720.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2263bddb67537aef1d3f281d85e09c1a542491a2
--- /dev/null
+++ b/checkpoints/model_weights_000079134720.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4d71339a1007a28a60ea5a1c34f7b337238e67db2fe9a76cd0c432c93dcd702
+size 284687203
diff --git a/checkpoints/model_weights_000081920000.pt b/checkpoints/model_weights_000081920000.pt
index 1c4f87e06e74d1b9dca21eae02f383372d869f72..aa6c1b4b940fe2108808ec99ba34afd122887953 100644
--- a/checkpoints/model_weights_000081920000.pt
+++ b/checkpoints/model_weights_000081920000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d1162bfd5bd1e15bce09767ece8e48a3a79a2adb81f37e505bfea386cb10927
-size 234226143
+oid sha256:1e5cc965285861e6cc6348e58d5efd5e7dafbbbc0e15fb388e7567f72c7e9c7d
+size 284687203
diff --git a/checkpoints/model_weights_000087048192.pt b/checkpoints/model_weights_000087048192.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d20a031cb4bc9dffcb38870e48364e0e6203e7d
--- /dev/null
+++ b/checkpoints/model_weights_000087048192.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d24da98febe12ffbbcb05bf2400ffb07892ba67081d88b673a1c379cda40ec3
+size 284687203
diff --git a/checkpoints/model_weights_000090112000.pt b/checkpoints/model_weights_000090112000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e71588d7212e0b8a37665c101be043ff6f7dd1e
--- /dev/null
+++ b/checkpoints/model_weights_000090112000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f1c5d860ba313ec361bd8f55ff6499c9ece890c1569bd0a6e253d65725b658
+size 284687203
diff --git a/checkpoints/model_weights_000095748096.pt b/checkpoints/model_weights_000095748096.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff0d69994f180ae91f475e8ec12c986bc1fa2d39
--- /dev/null
+++ b/checkpoints/model_weights_000095748096.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:296fb584c5a4358102d1793c6d85db6bf113b8ff22682079b076e61d519ec9d1
+size 284687203
diff --git a/checkpoints/model_weights_000098304000.pt b/checkpoints/model_weights_000098304000.pt
index 627f8009cd1120c0153672ab90a79c47ea0a0e92..1f9d61bfc28e8b7dd5eb38378ce97ebdec2e3ea9 100644
--- a/checkpoints/model_weights_000098304000.pt
+++ b/checkpoints/model_weights_000098304000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecb9dc3ca0c5d84882a29242eb11b155f98f06a37697a85a1548e86e237d89bf
-size 234226143
+oid sha256:4d39b3954672ba6327acb792d9645e958b95085d734188a8badc42fbd2b08c08
+size 284687203
diff --git a/checkpoints/model_weights_000105316352.pt b/checkpoints/model_weights_000105316352.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a94ee3bb0f75a89be709a396959d4ee26c5cce2
--- /dev/null
+++ b/checkpoints/model_weights_000105316352.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0901baaf20ff2a0e5c43eb3d98fca5bb1e9917d2a6be0ca623f3b2974019a6a
+size 284687203
diff --git a/checkpoints/model_weights_000106496000.pt b/checkpoints/model_weights_000106496000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab4ef7b949cd88d26c5d2c5130a511b4dc6475fe
--- /dev/null
+++ b/checkpoints/model_weights_000106496000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59050271dbcd19081d43346202396a8f3f05ccd486b32b5f3e6f7078d279e2d
+size 284687203
diff --git a/checkpoints/model_weights_000114688000.pt b/checkpoints/model_weights_000114688000.pt
index 9bfba36ee41af7c3f26d1e1aef2795a7ccbd7c10..a69f1d4cffe2c8430140e574504210986798c7da 100644
--- a/checkpoints/model_weights_000114688000.pt
+++ b/checkpoints/model_weights_000114688000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f21e5848c706c29bb35ef8117b4c4b66921cf45313bbe248fe3c36abd0b761f
-size 234226143
+oid sha256:24642db8b874beafee44da75ad14c9218d25a158c2e9033c5a28c3007c04600a
+size 284687203
diff --git a/checkpoints/model_weights_000115851264.pt b/checkpoints/model_weights_000115851264.pt
new file mode 100644
index 0000000000000000000000000000000000000000..13ab83a2a4876f75635e61061b1bbef91cc78f94
--- /dev/null
+++ b/checkpoints/model_weights_000115851264.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f0422bff7df5fcbe6f6a036fe98b8a5df77549ba37290eb750442153623f093
+size 284687203
diff --git a/checkpoints/model_weights_000122880000.pt b/checkpoints/model_weights_000122880000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ece85fc44ee43d4bb74f8561e7c1ccbc204509d5
--- /dev/null
+++ b/checkpoints/model_weights_000122880000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b70c23aa5b6c6861cec75239edec67919b010f1b5570f030a6357929106e88ac
+size 284687203
diff --git a/checkpoints/model_weights_000127434752.pt b/checkpoints/model_weights_000127434752.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0382ec97bcc38f538a9c80fcf6369d2af84568a2
--- /dev/null
+++ b/checkpoints/model_weights_000127434752.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c21d316a1ad329100af89561d4c0a212027a32c8efb6d422dc8b49c15af06cb
+size 284687203
diff --git a/checkpoints/model_weights_000131072000.pt b/checkpoints/model_weights_000131072000.pt
index 7ef3abed78bf8dcaf1641ee1de82a52ed1f47faf..9dd3c99e65863e109bfd467baa406b80a3ecb2a4 100644
--- a/checkpoints/model_weights_000131072000.pt
+++ b/checkpoints/model_weights_000131072000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5aa791b7e23c29d6b470ee3af51ce73cae658522112be61f4fafe23f728682fe
-size 234226143
+oid sha256:5ec9bb020e86634dccc22e9ad92e7d9f4242a638bc38ab0db0383fa8ac92877b
+size 284687203
diff --git a/checkpoints/model_weights_000139264000.pt b/checkpoints/model_weights_000139264000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60d33c7b5c9aa8765b10c56cbbeef3a702b3913f
--- /dev/null
+++ b/checkpoints/model_weights_000139264000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4816a4404720fc9732e24fd30c3bbf81830139bffcbf7a13ffaebdea6dc11e73
+size 284687203
diff --git a/checkpoints/model_weights_000140181504.pt b/checkpoints/model_weights_000140181504.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c089565883415f177c94a0ce9a0f7c111e5e89ca
--- /dev/null
+++ b/checkpoints/model_weights_000140181504.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08f2604d7aa96b6801c65dd699c10bad12f464632c6f929171a5244bb58f031d
+size 284687203
diff --git a/checkpoints/model_weights_000147456000.pt b/checkpoints/model_weights_000147456000.pt
index 0f5efe6f5b8e2046c6261ba40b9f5da01b5ef521..1ca29278e622fcec1c95ad33667abf73c5ac345e 100644
--- a/checkpoints/model_weights_000147456000.pt
+++ b/checkpoints/model_weights_000147456000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d1b493766132ed4e9242e9be2302f4a4e4b0cea52ab5654cb7a7ff5722537b9
-size 234226143
+oid sha256:196925b562fb0e8d2beeec1387f4aa161724617b1b7167125b640b593487f509
+size 284687203
diff --git a/checkpoints/model_weights_000154206208.pt b/checkpoints/model_weights_000154206208.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5098f7add8ee4513396f154a02a4610af774460
--- /dev/null
+++ b/checkpoints/model_weights_000154206208.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77e550c53927c094ba937a6dc00b255bdc5865f6cbc17ba3f4922a762c898a64
+size 284687203
diff --git a/checkpoints/model_weights_000155648000.pt b/checkpoints/model_weights_000155648000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d047efa5810fb3f8f8783ab16ad76d7edd5bea4b
--- /dev/null
+++ b/checkpoints/model_weights_000155648000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5e52db46127f24a08522cd8618e5ace30d8260b05c1d639cba62bcbddfc29cd
+size 284687203
diff --git a/checkpoints/model_weights_000163840000.pt b/checkpoints/model_weights_000163840000.pt
index 9977bd5035510c1a0952adad2faa6332eeb733d9..737ff50d297c31b87a808c3178513512e1fb02ff 100644
--- a/checkpoints/model_weights_000163840000.pt
+++ b/checkpoints/model_weights_000163840000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47c01e1e0df8d086a8081f34224be6188bbb070f0fbd772995382f9768e05150
-size 234226143
+oid sha256:34fd0a241eb11771c76231347a47e1d51993ed610368b505ea7a9a7debbaace1
+size 284687203
diff --git a/checkpoints/model_weights_000169623552.pt b/checkpoints/model_weights_000169623552.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3667f121eabaac6231000ca1d68d8e170cdc80e0
--- /dev/null
+++ b/checkpoints/model_weights_000169623552.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:628149a7f4fa6caea4d80ad70955dcdfa449b107b0ca9c29bb9dd5c452454bf6
+size 284687203
diff --git a/checkpoints/model_weights_000172032000.pt b/checkpoints/model_weights_000172032000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6dcd5554e0cc02f707e58beed1ad2fa9d3c299e8
--- /dev/null
+++ b/checkpoints/model_weights_000172032000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d37bd3248d7a22b3b1ecf52fb3c89dbf03e9073b9afb1767928f68fd32fe8ca
+size 284687203
diff --git a/checkpoints/model_weights_000180224000.pt b/checkpoints/model_weights_000180224000.pt
index 0891871d0f07c14a1a53bd7b5a45d7452e00d850..8f300a2d5379a38c23caf64779c11463f57ca016 100644
--- a/checkpoints/model_weights_000180224000.pt
+++ b/checkpoints/model_weights_000180224000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:689819a3ebd7a0252d9afba99726a081c4547f9e6453768639ed2efa2fa6e6ae
-size 234226143
+oid sha256:b20f535cb26260ca3cbb0a3a06005f60b11caf81cf51519ca91f10b5a9015dd0
+size 284687203
diff --git a/checkpoints/model_weights_000186580992.pt b/checkpoints/model_weights_000186580992.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d9c0f83e4238978774c28725aa55346fb2d68
--- /dev/null
+++ b/checkpoints/model_weights_000186580992.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8991c3943ee2545633fb60a9bcbd8831188a95b53df5253f288cbcf979bfa097
+size 284687203
diff --git a/checkpoints/model_weights_000188416000.pt b/checkpoints/model_weights_000188416000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fde68cf2511df64a9dec7e8025ffa4d30ed2f90
--- /dev/null
+++ b/checkpoints/model_weights_000188416000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf5d2360f6535ebb50ed6584a84dabc958d4b8efb794f32c8ccd8e5f386ba7d2
+size 284687203
diff --git a/checkpoints/model_weights_000196608000.pt b/checkpoints/model_weights_000196608000.pt
index ca57ea82e1a430e5005ff49ab5757fd57686f99e..2baefcb60540b66d737fa88ef932314a9bcdc88c 100644
--- a/checkpoints/model_weights_000196608000.pt
+++ b/checkpoints/model_weights_000196608000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1897fb0e3d6a21cf439e83f645f75ac227f95fff8bb408595c02b4791bf3fdf8
-size 234226143
+oid sha256:e3a315c7f76678e7b4980f41f268af267201657fb9787514f776e8b84e68c7d0
+size 284687203
diff --git a/checkpoints/model_weights_000198361088.pt b/checkpoints/model_weights_000198361088.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a9b64f22b257eba59ed4c852ef824b640ac8ffb
--- /dev/null
+++ b/checkpoints/model_weights_000198361088.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2efe21ec042d5c5f48bd2591e9cfa94ce8109e3fd73d9ece6bb9ab85cb8e12
+size 284687203
diff --git a/checkpoints/model_weights_000198688768.pt b/checkpoints/model_weights_000198688768.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6aa927bdaa18837819c86c20549d4da1050fca5
--- /dev/null
+++ b/checkpoints/model_weights_000198688768.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ffaeee4ce91fe968ac3dc6753b8ed7f68d3a4195c73ddd87d9b51e2b461bee8
+size 284687203
diff --git a/checkpoints/model_weights_000199016448.pt b/checkpoints/model_weights_000199016448.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed99b96242ee33b09485756728f909ea3c308f43
--- /dev/null
+++ b/checkpoints/model_weights_000199016448.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97b1ea058a4c69260d4baf9392327918b67f664add8253e7afe6a365155ed07b
+size 284687203
diff --git a/checkpoints/model_weights_000199344128.pt b/checkpoints/model_weights_000199344128.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e699fbeca6f47acd5b158f9633222316d4bbc940
--- /dev/null
+++ b/checkpoints/model_weights_000199344128.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cadb4fdd89d7c158ab5bad960fc4472d2715354178a02935fcc0f15bff19170b
+size 284687203
diff --git a/checkpoints/model_weights_000199671808.pt b/checkpoints/model_weights_000199671808.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c375b4f62df54a7628e0bbd6e1b478259041097
--- /dev/null
+++ b/checkpoints/model_weights_000199671808.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee1cb2cc10b54c401f00c6d229f42f54be29117394796d3ced8ab1259b800ebc
+size 284687203
diff --git a/checkpoints/model_weights_000199983104.pt b/checkpoints/model_weights_000199983104.pt
new file mode 100644
index 0000000000000000000000000000000000000000..95dff8729b350a384aaf97f53955f8bdd52053a4
--- /dev/null
+++ b/checkpoints/model_weights_000199983104.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04ac3de2ebf9f977709458a61e657dea3f58c25eb44d2dd080cfb39d23c017c3
+size 284687203
diff --git a/config.toml b/config.toml
index 80db26854e37fd5bff8adf0d0bc35e8b9cc82360..c5f97c04febc17e726433a1110e02fe96e423bd6 100644
--- a/config.toml
+++ b/config.toml
@@ -1,6 +1,6 @@
-model_name = "pile_llama_replace_17367_new"
+model_name = "pile_llama_replace_17367_L8_1024"
 dataset_name = "eoinf/PL_Replace17367_L2_alldataset"
-n_layers = 8
+n_layers = 12
 d_model = 512
 d_mlp = 2048
 d_head = 64
@@ -13,7 +13,7 @@ d_vocab = 32000
 seed = 10
 device = "cuda"
 use_bfloat16_matmul = false
-batch_size_per_device = 32
+batch_size_per_device = 16
 n_devices = 1
 batches_per_step = 1
 max_tokens = 200000000
diff --git a/latest_checkpoint.pt b/latest_checkpoint.pt
index 6ebad479267640af5984618443d6d88f363c9c5a..10c15aa12aec90d7ca6087993760697228c52ad2 100644
--- a/latest_checkpoint.pt
+++ b/latest_checkpoint.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5afdfec5e54636ba35ba3fe4adcc0c626b47c8fc61b2b5c7ee57f4a1469efeb6
-size 234224683
+oid sha256:1ef12c55e4064f31928e0e94faec665fbe234169ab01323103f9ed95bbb6a5ec
+size 284685195
diff --git a/latest_metadata.json b/latest_metadata.json
index 389eb4b3f672e2d2c62d2b59ee77e494577e09db..b89a001b93d5048c0cb80071284c2e6c56a8fc4e 100644
--- a/latest_metadata.json
+++ b/latest_metadata.json
@@ -1 +1 @@
-{"step": 6102, "tokens_seen": 199950336, "config": {"model_name": "pile_llama_replace_17367_new", "n_layers": 8, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 3.1541889009453166}
\ No newline at end of file
+{"step": 12206, "tokens_seen": 199983104, "config": {"model_name": "pile_llama_replace_17367_L8_1024", "n_layers": 12, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 32000, "dataset_name": "eoinf/PL_Replace17367_L2_alldataset", "tokenizer_name": "", "seed": 10, "data_seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 16, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 16, "tokens_per_step": 16384, "warmup_steps": 1831, "max_steps": 12207}, "train_loss_ewma": 3.1248199771775043}
\ No newline at end of file
diff --git a/latest_optimizer.pt b/latest_optimizer.pt
index cdd7d061852d1e6939136e551780fccd6f5382a0..9748dc269d4f18ccc0dd49397d563ee211ad070a 100644
--- a/latest_optimizer.pt
+++ b/latest_optimizer.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68d4a23c5ae1feb0c0677d5fc0fff682ffbfff0aada4fbc60104a0057ee50435
-size 468470963
+oid sha256:01cc58843d7c9e2b10d7894a42f6d983d75e1d679af4e2cec69a29589944db11
+size 569402163
diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log
index d0ef417f5fa4b559f89e60d07c150e2e3f061f99..acf4434179822f6b9a52fc76ffbe4efd3446278f 100644
--- a/wandb/debug-internal.log
+++ b/wandb/debug-internal.log
@@ -1,12 +1,12 @@
-{"time":"2026-03-19T06:35:19.214150803Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
-{"time":"2026-03-19T06:35:19.454890439Z","level":"INFO","msg":"stream: created new stream","id":"29lbcxak"}
-{"time":"2026-03-19T06:35:19.454977401Z","level":"INFO","msg":"stream: started","id":"29lbcxak"}
-{"time":"2026-03-19T06:35:19.455055746Z","level":"INFO","msg":"sender: started","stream_id":"29lbcxak"}
-{"time":"2026-03-19T06:35:19.455093161Z","level":"INFO","msg":"writer: started","stream_id":"29lbcxak"}
-{"time":"2026-03-19T06:35:19.455093346Z","level":"INFO","msg":"handler: started","stream_id":"29lbcxak"}
-{"time":"2026-03-19T08:40:04.994899105Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":0.001043706}],"total_operations":1}}
-{"time":"2026-03-19T08:40:05.630105109Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2026-03-19T08:40:08.650518881Z","level":"INFO","msg":"stream: closing","id":"29lbcxak"}
-{"time":"2026-03-19T08:40:08.650561694Z","level":"INFO","msg":"handler: closed","stream_id":"29lbcxak"}
-{"time":"2026-03-19T08:40:08.650605449Z","level":"INFO","msg":"sender: closed","stream_id":"29lbcxak"}
-{"time":"2026-03-19T08:40:08.650623595Z","level":"INFO","msg":"stream: closed","id":"29lbcxak"}
+{"time":"2026-03-19T09:10:54.518147627Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2026-03-19T09:10:54.728754949Z","level":"INFO","msg":"stream: created new stream","id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.728795071Z","level":"INFO","msg":"stream: started","id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.729757806Z","level":"INFO","msg":"sender: started","stream_id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.72979291Z","level":"INFO","msg":"handler: started","stream_id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.729814605Z","level":"INFO","msg":"writer: started","stream_id":"lisp43b6"}
+{"time":"2026-03-19T12:06:02.518903145Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading data","runtime_seconds":9.420394008},{"desc":"updating run metadata","runtime_seconds":0.001821517}],"total_operations":2}}
+{"time":"2026-03-19T12:06:03.072131148Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-03-19T12:06:12.889325955Z","level":"INFO","msg":"stream: closing","id":"lisp43b6"}
+{"time":"2026-03-19T12:06:12.889384907Z","level":"INFO","msg":"handler: closed","stream_id":"lisp43b6"}
+{"time":"2026-03-19T12:06:12.889456031Z","level":"INFO","msg":"sender: closed","stream_id":"lisp43b6"}
+{"time":"2026-03-19T12:06:12.889466313Z","level":"INFO","msg":"stream: closed","id":"lisp43b6"}
diff --git a/wandb/debug.log b/wandb/debug.log
index cfed7b25c4e3c86ac39418d92b12214e64089747..6341541a524f3bf7b2378d00f86247eec82d17aa 100644
--- a/wandb/debug.log
+++ b/wandb/debug.log
@@ -1,26 +1,26 @@
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_setup.py:_flush():81] Configure stats pid to 2015
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/model/wandb/settings
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_063518-29lbcxak/logs/debug.log
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_063518-29lbcxak/logs/debug-internal.log
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_init.py:init():813] calling init triggers
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
-config: {'model_name': 'pile_llama_replace_17367_new', 'n_layers': 8, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 32000, 'dataset_name': 'eoinf/PL_Replace17367_L2_alldataset', 'tokenizer_name': '', 'seed': 10, 'data_seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 32, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.002, 'lr_vector': 0.001, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 32, 'tokens_per_step': 32768, 'warmup_steps': 915, 'max_steps': 6103, '_wandb': {}}
-2026-03-19 06:35:18,696 INFO    MainThread:2015 [wandb_init.py:init():854] starting backend
-2026-03-19 06:35:19,200 INFO    MainThread:2015 [wandb_init.py:init():857] sending inform_init request
-2026-03-19 06:35:19,210 INFO    MainThread:2015 [wandb_init.py:init():865] backend started and connected
-2026-03-19 06:35:19,211 INFO    MainThread:2015 [wandb_init.py:init():936] updated telemetry
-2026-03-19 06:35:19,221 INFO    MainThread:2015 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
-2026-03-19 06:35:20,106 INFO    MainThread:2015 [wandb_init.py:init():1011] starting run threads in backend
-2026-03-19 06:35:20,927 INFO    MainThread:2015 [wandb_run.py:_console_start():2506] atexit reg
-2026-03-19 06:35:20,927 INFO    MainThread:2015 [wandb_run.py:_redirect():2354] redirect: wrap_raw
-2026-03-19 06:35:20,928 INFO    MainThread:2015 [wandb_run.py:_redirect():2423] Wrapping output streams.
-2026-03-19 06:35:20,928 INFO    MainThread:2015 [wandb_run.py:_redirect():2446] Redirects installed.
-2026-03-19 06:35:20,941 INFO    MainThread:2015 [wandb_init.py:init():1049] run started, returning control to user process
-2026-03-19 08:40:04,989 INFO    MainThread:2015 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/29lbcxak
-2026-03-19 08:40:04,993 INFO    MainThread:2015 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0
-2026-03-19 08:40:04,993 INFO    MainThread:2015 [wandb_run.py:_restore():2453] restore
-2026-03-19 08:40:04,993 INFO    MainThread:2015 [wandb_run.py:_restore():2459] restore done
-2026-03-19 08:40:08,649 INFO    MainThread:2015 [wandb_run.py:_footer_sync_info():3867] logging synced files
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Configure stats pid to 678
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/model/wandb/settings
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_091054-lisp43b6/logs/debug.log
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_091054-lisp43b6/logs/debug-internal.log
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:init():813] calling init triggers
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'pile_llama_replace_17367_L8_1024', 'n_layers': 12, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 32000, 'dataset_name': 'eoinf/PL_Replace17367_L2_alldataset', 'tokenizer_name': '', 'seed': 10, 'data_seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 16, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.002, 'lr_vector': 0.001, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 16, 'tokens_per_step': 16384, 'warmup_steps': 1831, 'max_steps': 12207, '_wandb': {}}
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:init():854] starting backend
+2026-03-19 09:10:54,512 INFO    MainThread:678 [wandb_init.py:init():857] sending inform_init request
+2026-03-19 09:10:54,516 INFO    MainThread:678 [wandb_init.py:init():865] backend started and connected
+2026-03-19 09:10:54,517 INFO    MainThread:678 [wandb_init.py:init():936] updated telemetry
+2026-03-19 09:10:54,557 INFO    MainThread:678 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2026-03-19 09:10:55,330 INFO    MainThread:678 [wandb_init.py:init():1011] starting run threads in backend
+2026-03-19 09:10:55,439 INFO    MainThread:678 [wandb_run.py:_console_start():2506] atexit reg
+2026-03-19 09:10:55,440 INFO    MainThread:678 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2026-03-19 09:10:55,440 INFO    MainThread:678 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2026-03-19 09:10:55,440 INFO    MainThread:678 [wandb_run.py:_redirect():2446] Redirects installed.
+2026-03-19 09:10:55,442 INFO    MainThread:678 [wandb_init.py:init():1049] run started, returning control to user process
+2026-03-19 12:06:02,510 INFO    MainThread:678 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/lisp43b6
+2026-03-19 12:06:02,516 INFO    MainThread:678 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0
+2026-03-19 12:06:02,517 INFO    MainThread:678 [wandb_run.py:_restore():2453] restore
+2026-03-19 12:06:02,517 INFO    MainThread:678 [wandb_run.py:_restore():2459] restore done
+2026-03-19 12:06:12,886 INFO    MainThread:678 [wandb_run.py:_footer_sync_info():3867] logging synced files
diff --git a/wandb/run-20260319_091054-lisp43b6/files/config.yaml b/wandb/run-20260319_091054-lisp43b6/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f4f328c46d499e444290a68e2565cee17f60632c
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/files/config.yaml
@@ -0,0 +1,140 @@
+_wandb:
+    value:
+        cli_version: 0.21.4
+        e:
+            jdofomlxripi823w2v6uwgggyst0ywfm:
+                cpu_count: 8
+                cpu_count_logical: 8
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "262240792576"
+                        used: "152260071424"
+                email: tzfof8@gmail.com
+                executable: /notebooks/toy_models/.toy_models_env/bin/python
+                git:
+                    commit: d722bb952956265d0387df9c35a76703a66824ec
+                    remote: https://github.com/jgroh3/toy_models.git
+                gpu: NVIDIA RTX A6000
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 10752
+                      memoryTotal: "51527024640"
+                      name: NVIDIA RTX A6000
+                      uuid: GPU-ec83d6f5-8082-3e0c-f60c-d8e819df4844
+                host: nmvyvu5qhg
+                memory:
+                    total: "47332843520"
+                os: Linux-5.19.0-45-generic-x86_64-with-glibc2.35
+                program: <python with no main file>
+                python: CPython 3.11.7
+                root: /notebooks/toy_models/model_training/model
+                startedAt: "2026-03-19T09:10:54.300170Z"
+                writerId: jdofomlxripi823w2v6uwgggyst0ywfm
+        m: []
+        python_version: 3.11.7
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.11.7
+            "5": 0.21.4
+            "6": 4.56.1
+            "12": 0.21.4
+            "13": linux-x86_64
+attn_only:
+    value: false
+batch_size:
+    value: 16
+batch_size_per_device:
+    value: 16
+batches_per_step:
+    value: 1
+checkpoint_interval:
+    value: 500
+checkpoint_interval_ratio:
+    value: 1.1
+d_head:
+    value: 64
+d_mlp:
+    value: 2048
+d_model:
+    value: 512
+d_vocab:
+    value: 32000
+data_seed:
+    value: 10
+dataset_name:
+    value: eoinf/PL_Replace17367_L2_alldataset
+device:
+    value: cuda
+grad_norm_clip:
+    value: 1
+init_range:
+    value: 0.02
+layer_norm_eps:
+    value: 1e-05
+log_interval:
+    value: 25
+lr_hidden:
+    value: 0.002
+lr_schedule:
+    value: constant_with_warmup
+lr_vector:
+    value: 0.001
+max_steps:
+    value: 12207
+max_tokens:
+    value: 200000000
+model_name:
+    value: pile_llama_replace_17367_L8_1024
+n_ctx:
+    value: 1024
+n_devices:
+    value: 1
+n_heads:
+    value: 8
+n_layers:
+    value: 12
+save_checkpoints:
+    value: true
+save_log_checkpoints:
+    value: true
+seed:
+    value: 10
+tokenizer_name:
+    value: ""
+tokens_per_step:
+    value: 16384
+train_loss_moving_average_beta:
+    value: 0.99
+use_bfloat16_matmul:
+    value: false
+use_wandb:
+    value: true
+warmup_steps:
+    value: 1831
+warmup_tokens:
+    value: 30000000
+weight_decay:
+    value: 0.05
diff --git a/wandb/run-20260319_091054-lisp43b6/files/output.log b/wandb/run-20260319_091054-lisp43b6/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..7908047c696ed0b28b2a983db2709e31fb36c804
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/files/output.log
@@ -0,0 +1,496 @@
+Training on cuda
+Model: 12L, 512d, 8h
+Max steps: 12,207, Max tokens: 200,000,000
+Warmup steps: 1,831, Warmup tokens: 30,000,000
+Batch size per device: 16
+Context length: 1024
+Learning rates - Hidden: 0.002, Vector: 0.001
+                                                                                                                                                                        
+Step 25 | Tokens: 409,600 | Train Loss EWMA: 10.4013 | Learning Rate: 0.000027 | Progress: 0.00205
+Step 50 | Tokens: 819,200 | Train Loss EWMA: 10.1837 | Learning Rate: 0.000055 | Progress: 0.00410
+Step 75 | Tokens: 1,228,800 | Train Loss EWMA: 9.8660 | Learning Rate: 0.000082 | Progress: 0.00614
+Step 100 | Tokens: 1,638,400 | Train Loss EWMA: 9.4737 | Learning Rate: 0.000109 | Progress: 0.00819
+Step 125 | Tokens: 2,048,000 | Train Loss EWMA: 9.0346 | Learning Rate: 0.000137 | Progress: 0.01024
+Step 150 | Tokens: 2,457,600 | Train Loss EWMA: 8.5790 | Learning Rate: 0.000164 | Progress: 0.01229
+Step 175 | Tokens: 2,867,200 | Train Loss EWMA: 8.1574 | Learning Rate: 0.000191 | Progress: 0.01434
+Step 200 | Tokens: 3,276,800 | Train Loss EWMA: 7.8043 | Learning Rate: 0.000218 | Progress: 0.01638
+Step 225 | Tokens: 3,686,400 | Train Loss EWMA: 7.5125 | Learning Rate: 0.000246 | Progress: 0.01843
+Step 250 | Tokens: 4,096,000 | Train Loss EWMA: 7.2410 | Learning Rate: 0.000273 | Progress: 0.02048
+Step 275 | Tokens: 4,505,600 | Train Loss EWMA: 6.9942 | Learning Rate: 0.000300 | Progress: 0.02253
+Step 300 | Tokens: 4,915,200 | Train Loss EWMA: 6.7891 | Learning Rate: 0.000328 | Progress: 0.02458
+Step 325 | Tokens: 5,324,800 | Train Loss EWMA: 6.6214 | Learning Rate: 0.000355 | Progress: 0.02662
+Step 350 | Tokens: 5,734,400 | Train Loss EWMA: 6.4692 | Learning Rate: 0.000382 | Progress: 0.02867
+Step 375 | Tokens: 6,144,000 | Train Loss EWMA: 6.3360 | Learning Rate: 0.000410 | Progress: 0.03072
+Step 400 | Tokens: 6,553,600 | Train Loss EWMA: 6.2263 | Learning Rate: 0.000437 | Progress: 0.03277
+Step 425 | Tokens: 6,963,200 | Train Loss EWMA: 6.1299 | Learning Rate: 0.000464 | Progress: 0.03482
+Step 450 | Tokens: 7,372,800 | Train Loss EWMA: 6.0353 | Learning Rate: 0.000492 | Progress: 0.03686
+Step 475 | Tokens: 7,782,400 | Train Loss EWMA: 5.9720 | Learning Rate: 0.000519 | Progress: 0.03891
+Step 500 | Tokens: 8,192,000 | Train Loss EWMA: 5.9094 | Learning Rate: 0.000546 | Progress: 0.04096
+Step 525 | Tokens: 8,601,600 | Train Loss EWMA: 5.8475 | Learning Rate: 0.000573 | Progress: 0.04301
+Step 550 | Tokens: 9,011,200 | Train Loss EWMA: 5.7907 | Learning Rate: 0.000601 | Progress: 0.04506
+Step 575 | Tokens: 9,420,800 | Train Loss EWMA: 5.7364 | Learning Rate: 0.000628 | Progress: 0.04710
+Step 600 | Tokens: 9,830,400 | Train Loss EWMA: 5.6822 | Learning Rate: 0.000655 | Progress: 0.04915
+Step 625 | Tokens: 10,240,000 | Train Loss EWMA: 5.6351 | Learning Rate: 0.000683 | Progress: 0.05120
+Step 650 | Tokens: 10,649,600 | Train Loss EWMA: 5.5952 | Learning Rate: 0.000710 | Progress: 0.05325
+Step 675 | Tokens: 11,059,200 | Train Loss EWMA: 5.5585 | Learning Rate: 0.000737 | Progress: 0.05530
+Step 700 | Tokens: 11,468,800 | Train Loss EWMA: 5.5204 | Learning Rate: 0.000765 | Progress: 0.05734
+Step 725 | Tokens: 11,878,400 | Train Loss EWMA: 5.4841 | Learning Rate: 0.000792 | Progress: 0.05939
+Step 750 | Tokens: 12,288,000 | Train Loss EWMA: 5.4608 | Learning Rate: 0.000819 | Progress: 0.06144
+Step 775 | Tokens: 12,697,600 | Train Loss EWMA: 5.4479 | Learning Rate: 0.000847 | Progress: 0.06349
+Step 800 | Tokens: 13,107,200 | Train Loss EWMA: 5.4132 | Learning Rate: 0.000874 | Progress: 0.06554
+Step 825 | Tokens: 13,516,800 | Train Loss EWMA: 5.3763 | Learning Rate: 0.000901 | Progress: 0.06758
+Step 850 | Tokens: 13,926,400 | Train Loss EWMA: 5.3532 | Learning Rate: 0.000928 | Progress: 0.06963
+Step 875 | Tokens: 14,336,000 | Train Loss EWMA: 5.3245 | Learning Rate: 0.000956 | Progress: 0.07168
+Step 900 | Tokens: 14,745,600 | Train Loss EWMA: 5.3134 | Learning Rate: 0.000983 | Progress: 0.07373
+Step 925 | Tokens: 15,155,200 | Train Loss EWMA: 5.2901 | Learning Rate: 0.001010 | Progress: 0.07578
+Step 950 | Tokens: 15,564,800 | Train Loss EWMA: 5.2667 | Learning Rate: 0.001038 | Progress: 0.07782
+Step 975 | Tokens: 15,974,400 | Train Loss EWMA: 5.2683 | Learning Rate: 0.001065 | Progress: 0.07987
+Step 1,000 | Tokens: 16,384,000 | Train Loss EWMA: 5.2575 | Learning Rate: 0.001092 | Progress: 0.08192
+Step 1,025 | Tokens: 16,793,600 | Train Loss EWMA: 5.2215 | Learning Rate: 0.001120 | Progress: 0.08397
+Step 1,050 | Tokens: 17,203,200 | Train Loss EWMA: 5.2106 | Learning Rate: 0.001147 | Progress: 0.08602
+Step 1,075 | Tokens: 17,612,800 | Train Loss EWMA: 5.1894 | Learning Rate: 0.001174 | Progress: 0.08806
+Step 1,100 | Tokens: 18,022,400 | Train Loss EWMA: 5.1590 | Learning Rate: 0.001202 | Progress: 0.09011
+Step 1,125 | Tokens: 18,432,000 | Train Loss EWMA: 5.1288 | Learning Rate: 0.001229 | Progress: 0.09216
+Step 1,150 | Tokens: 18,841,600 | Train Loss EWMA: 5.1122 | Learning Rate: 0.001256 | Progress: 0.09421
+Step 1,175 | Tokens: 19,251,200 | Train Loss EWMA: 5.0993 | Learning Rate: 0.001283 | Progress: 0.09626
+Step 1,200 | Tokens: 19,660,800 | Train Loss EWMA: 5.0859 | Learning Rate: 0.001311 | Progress: 0.09830
+Step 1,225 | Tokens: 20,070,400 | Train Loss EWMA: 5.0766 | Learning Rate: 0.001338 | Progress: 0.10035
+Step 1,250 | Tokens: 20,480,000 | Train Loss EWMA: 5.0518 | Learning Rate: 0.001365 | Progress: 0.10240
+Step 1,275 | Tokens: 20,889,600 | Train Loss EWMA: 5.0477 | Learning Rate: 0.001393 | Progress: 0.10445
+Step 1,300 | Tokens: 21,299,200 | Train Loss EWMA: 5.0499 | Learning Rate: 0.001420 | Progress: 0.10650
+Step 1,325 | Tokens: 21,708,800 | Train Loss EWMA: 5.0495 | Learning Rate: 0.001447 | Progress: 0.10854
+Step 1,350 | Tokens: 22,118,400 | Train Loss EWMA: 5.0256 | Learning Rate: 0.001475 | Progress: 0.11059
+Step 1,375 | Tokens: 22,528,000 | Train Loss EWMA: 5.0155 | Learning Rate: 0.001502 | Progress: 0.11264
+Step 1,400 | Tokens: 22,937,600 | Train Loss EWMA: 5.0074 | Learning Rate: 0.001529 | Progress: 0.11469
+Step 1,425 | Tokens: 23,347,200 | Train Loss EWMA: 4.9936 | Learning Rate: 0.001557 | Progress: 0.11674
+Step 1,450 | Tokens: 23,756,800 | Train Loss EWMA: 4.9785 | Learning Rate: 0.001584 | Progress: 0.11878
+Step 1,475 | Tokens: 24,166,400 | Train Loss EWMA: 4.9581 | Learning Rate: 0.001611 | Progress: 0.12083
+Step 1,500 | Tokens: 24,576,000 | Train Loss EWMA: 4.9309 | Learning Rate: 0.001638 | Progress: 0.12288
+Step 1,525 | Tokens: 24,985,600 | Train Loss EWMA: 4.9296 | Learning Rate: 0.001666 | Progress: 0.12493
+Step 1,550 | Tokens: 25,395,200 | Train Loss EWMA: 4.9230 | Learning Rate: 0.001693 | Progress: 0.12698
+Step 1,575 | Tokens: 25,804,800 | Train Loss EWMA: 4.9127 | Learning Rate: 0.001720 | Progress: 0.12902
+Step 1,600 | Tokens: 26,214,400 | Train Loss EWMA: 4.9063 | Learning Rate: 0.001748 | Progress: 0.13107
+Step 1,625 | Tokens: 26,624,000 | Train Loss EWMA: 4.8864 | Learning Rate: 0.001775 | Progress: 0.13312
+Step 1,650 | Tokens: 27,033,600 | Train Loss EWMA: 4.8710 | Learning Rate: 0.001802 | Progress: 0.13517
+Step 1,675 | Tokens: 27,443,200 | Train Loss EWMA: 4.8719 | Learning Rate: 0.001830 | Progress: 0.13722
+Step 1,700 | Tokens: 27,852,800 | Train Loss EWMA: 4.8553 | Learning Rate: 0.001857 | Progress: 0.13926
+Step 1,725 | Tokens: 28,262,400 | Train Loss EWMA: 4.8366 | Learning Rate: 0.001884 | Progress: 0.14131
+Step 1,750 | Tokens: 28,672,000 | Train Loss EWMA: 4.8291 | Learning Rate: 0.001912 | Progress: 0.14336
+Step 1,775 | Tokens: 29,081,600 | Train Loss EWMA: 4.8190 | Learning Rate: 0.001939 | Progress: 0.14541
+Step 1,800 | Tokens: 29,491,200 | Train Loss EWMA: 4.8087 | Learning Rate: 0.001966 | Progress: 0.14746
+Step 1,825 | Tokens: 29,900,800 | Train Loss EWMA: 4.7994 | Learning Rate: 0.001993 | Progress: 0.14950
+Step 1,850 | Tokens: 30,310,400 | Train Loss EWMA: 4.7851 | Learning Rate: 0.002000 | Progress: 0.15155
+Step 1,875 | Tokens: 30,720,000 | Train Loss EWMA: 4.7727 | Learning Rate: 0.002000 | Progress: 0.15360
+Step 1,900 | Tokens: 31,129,600 | Train Loss EWMA: 4.7632 | Learning Rate: 0.002000 | Progress: 0.15565
+Step 1,925 | Tokens: 31,539,200 | Train Loss EWMA: 4.7549 | Learning Rate: 0.002000 | Progress: 0.15770
+Step 1,950 | Tokens: 31,948,800 | Train Loss EWMA: 4.7348 | Learning Rate: 0.002000 | Progress: 0.15974
+Step 1,975 | Tokens: 32,358,400 | Train Loss EWMA: 4.7153 | Learning Rate: 0.002000 | Progress: 0.16179
+Step 2,000 | Tokens: 32,768,000 | Train Loss EWMA: 4.6952 | Learning Rate: 0.002000 | Progress: 0.16384
+Step 2,025 | Tokens: 33,177,600 | Train Loss EWMA: 4.6779 | Learning Rate: 0.002000 | Progress: 0.16589
+Step 2,050 | Tokens: 33,587,200 | Train Loss EWMA: 4.6605 | Learning Rate: 0.002000 | Progress: 0.16794
+Step 2,075 | Tokens: 33,996,800 | Train Loss EWMA: 4.6530 | Learning Rate: 0.002000 | Progress: 0.16998
+Step 2,100 | Tokens: 34,406,400 | Train Loss EWMA: 4.6442 | Learning Rate: 0.002000 | Progress: 0.17203
+Step 2,125 | Tokens: 34,816,000 | Train Loss EWMA: 4.6348 | Learning Rate: 0.002000 | Progress: 0.17408
+Step 2,150 | Tokens: 35,225,600 | Train Loss EWMA: 4.6279 | Learning Rate: 0.002000 | Progress: 0.17613
+Step 2,175 | Tokens: 35,635,200 | Train Loss EWMA: 4.6117 | Learning Rate: 0.002000 | Progress: 0.17818
+Step 2,200 | Tokens: 36,044,800 | Train Loss EWMA: 4.5996 | Learning Rate: 0.002000 | Progress: 0.18022
+Step 2,225 | Tokens: 36,454,400 | Train Loss EWMA: 4.5758 | Learning Rate: 0.002000 | Progress: 0.18227
+Step 2,250 | Tokens: 36,864,000 | Train Loss EWMA: 4.5589 | Learning Rate: 0.002000 | Progress: 0.18432
+Step 2,275 | Tokens: 37,273,600 | Train Loss EWMA: 4.5495 | Learning Rate: 0.002000 | Progress: 0.18637
+Step 2,300 | Tokens: 37,683,200 | Train Loss EWMA: 4.5409 | Learning Rate: 0.002000 | Progress: 0.18842
+Step 2,325 | Tokens: 38,092,800 | Train Loss EWMA: 4.5218 | Learning Rate: 0.002000 | Progress: 0.19046
+Step 2,350 | Tokens: 38,502,400 | Train Loss EWMA: 4.5074 | Learning Rate: 0.002000 | Progress: 0.19251
+Step 2,375 | Tokens: 38,912,000 | Train Loss EWMA: 4.4974 | Learning Rate: 0.002000 | Progress: 0.19456
+Step 2,400 | Tokens: 39,321,600 | Train Loss EWMA: 4.4956 | Learning Rate: 0.002000 | Progress: 0.19661
+Step 2,425 | Tokens: 39,731,200 | Train Loss EWMA: 4.4874 | Learning Rate: 0.002000 | Progress: 0.19866
+Step 2,450 | Tokens: 40,140,800 | Train Loss EWMA: 4.4781 | Learning Rate: 0.002000 | Progress: 0.20070
+Step 2,475 | Tokens: 40,550,400 | Train Loss EWMA: 4.4582 | Learning Rate: 0.002000 | Progress: 0.20275
+Step 2,500 | Tokens: 40,960,000 | Train Loss EWMA: 4.4452 | Learning Rate: 0.002000 | Progress: 0.20480
+Step 2,525 | Tokens: 41,369,600 | Train Loss EWMA: 4.4322 | Learning Rate: 0.002000 | Progress: 0.20685
+Step 2,550 | Tokens: 41,779,200 | Train Loss EWMA: 4.4230 | Learning Rate: 0.002000 | Progress: 0.20890
+Step 2,575 | Tokens: 42,188,800 | Train Loss EWMA: 4.4038 | Learning Rate: 0.002000 | Progress: 0.21094
+Step 2,600 | Tokens: 42,598,400 | Train Loss EWMA: 4.3969 | Learning Rate: 0.002000 | Progress: 0.21299
+Step 2,625 | Tokens: 43,008,000 | Train Loss EWMA: 4.3915 | Learning Rate: 0.002000 | Progress: 0.21504
+Step 2,650 | Tokens: 43,417,600 | Train Loss EWMA: 4.3833 | Learning Rate: 0.002000 | Progress: 0.21709
+Step 2,675 | Tokens: 43,827,200 | Train Loss EWMA: 4.3621 | Learning Rate: 0.002000 | Progress: 0.21914
+Step 2,700 | Tokens: 44,236,800 | Train Loss EWMA: 4.3445 | Learning Rate: 0.002000 | Progress: 0.22118
+Step 2,725 | Tokens: 44,646,400 | Train Loss EWMA: 4.3267 | Learning Rate: 0.002000 | Progress: 0.22323
+Step 2,750 | Tokens: 45,056,000 | Train Loss EWMA: 4.3226 | Learning Rate: 0.002000 | Progress: 0.22528
+Step 2,775 | Tokens: 45,465,600 | Train Loss EWMA: 4.3242 | Learning Rate: 0.002000 | Progress: 0.22733
+Step 2,800 | Tokens: 45,875,200 | Train Loss EWMA: 4.3057 | Learning Rate: 0.002000 | Progress: 0.22938
+Step 2,825 | Tokens: 46,284,800 | Train Loss EWMA: 4.3015 | Learning Rate: 0.002000 | Progress: 0.23142
+Step 2,850 | Tokens: 46,694,400 | Train Loss EWMA: 4.2943 | Learning Rate: 0.002000 | Progress: 0.23347
+Step 2,875 | Tokens: 47,104,000 | Train Loss EWMA: 4.2870 | Learning Rate: 0.002000 | Progress: 0.23552
+Step 2,900 | Tokens: 47,513,600 | Train Loss EWMA: 4.2833 | Learning Rate: 0.002000 | Progress: 0.23757
+Step 2,925 | Tokens: 47,923,200 | Train Loss EWMA: 4.2737 | Learning Rate: 0.002000 | Progress: 0.23962
+Step 2,950 | Tokens: 48,332,800 | Train Loss EWMA: 4.2654 | Learning Rate: 0.002000 | Progress: 0.24166
+Step 2,975 | Tokens: 48,742,400 | Train Loss EWMA: 4.2593 | Learning Rate: 0.002000 | Progress: 0.24371
+Step 3,000 | Tokens: 49,152,000 | Train Loss EWMA: 4.2490 | Learning Rate: 0.002000 | Progress: 0.24576
+Step 3,025 | Tokens: 49,561,600 | Train Loss EWMA: 4.2557 | Learning Rate: 0.002000 | Progress: 0.24781
+Step 3,050 | Tokens: 49,971,200 | Train Loss EWMA: 4.2546 | Learning Rate: 0.002000 | Progress: 0.24986
+Step 3,075 | Tokens: 50,380,800 | Train Loss EWMA: 4.2460 | Learning Rate: 0.002000 | Progress: 0.25190
+Step 3,100 | Tokens: 50,790,400 | Train Loss EWMA: 4.2283 | Learning Rate: 0.002000 | Progress: 0.25395
+Step 3,125 | Tokens: 51,200,000 | Train Loss EWMA: 4.2123 | Learning Rate: 0.002000 | Progress: 0.25600
+Step 3,150 | Tokens: 51,609,600 | Train Loss EWMA: 4.1958 | Learning Rate: 0.002000 | Progress: 0.25805
+Step 3,175 | Tokens: 52,019,200 | Train Loss EWMA: 4.1876 | Learning Rate: 0.002000 | Progress: 0.26010
+Step 3,200 | Tokens: 52,428,800 | Train Loss EWMA: 4.1654 | Learning Rate: 0.002000 | Progress: 0.26214
+Step 3,225 | Tokens: 52,838,400 | Train Loss EWMA: 4.1544 | Learning Rate: 0.002000 | Progress: 0.26419
+Step 3,250 | Tokens: 53,248,000 | Train Loss EWMA: 4.1313 | Learning Rate: 0.002000 | Progress: 0.26624
+Step 3,275 | Tokens: 53,657,600 | Train Loss EWMA: 4.1171 | Learning Rate: 0.002000 | Progress: 0.26829
+Step 3,300 | Tokens: 54,067,200 | Train Loss EWMA: 4.1087 | Learning Rate: 0.002000 | Progress: 0.27034
+Step 3,325 | Tokens: 54,476,800 | Train Loss EWMA: 4.0881 | Learning Rate: 0.002000 | Progress: 0.27238
+Step 3,350 | Tokens: 54,886,400 | Train Loss EWMA: 4.0756 | Learning Rate: 0.002000 | Progress: 0.27443
+Step 3,375 | Tokens: 55,296,000 | Train Loss EWMA: 4.0535 | Learning Rate: 0.002000 | Progress: 0.27648
+Step 3,400 | Tokens: 55,705,600 | Train Loss EWMA: 4.0370 | Learning Rate: 0.002000 | Progress: 0.27853
+Step 3,425 | Tokens: 56,115,200 | Train Loss EWMA: 4.0301 | Learning Rate: 0.002000 | Progress: 0.28058
+Step 3,450 | Tokens: 56,524,800 | Train Loss EWMA: 4.0216 | Learning Rate: 0.002000 | Progress: 0.28262
+Step 3,475 | Tokens: 56,934,400 | Train Loss EWMA: 4.0109 | Learning Rate: 0.002000 | Progress: 0.28467
+Step 3,500 | Tokens: 57,344,000 | Train Loss EWMA: 3.9845 | Learning Rate: 0.002000 | Progress: 0.28672
+Step 3,525 | Tokens: 57,753,600 | Train Loss EWMA: 3.9674 | Learning Rate: 0.002000 | Progress: 0.28877
+Step 3,550 | Tokens: 58,163,200 | Train Loss EWMA: 3.9478 | Learning Rate: 0.002000 | Progress: 0.29082
+Step 3,575 | Tokens: 58,572,800 | Train Loss EWMA: 3.9465 | Learning Rate: 0.002000 | Progress: 0.29286
+Step 3,600 | Tokens: 58,982,400 | Train Loss EWMA: 3.9427 | Learning Rate: 0.002000 | Progress: 0.29491
+Step 3,625 | Tokens: 59,392,000 | Train Loss EWMA: 3.9304 | Learning Rate: 0.002000 | Progress: 0.29696
+Step 3,650 | Tokens: 59,801,600 | Train Loss EWMA: 3.9059 | Learning Rate: 0.002000 | Progress: 0.29901
+Step 3,675 | Tokens: 60,211,200 | Train Loss EWMA: 3.9053 | Learning Rate: 0.002000 | Progress: 0.30106
+Step 3,700 | Tokens: 60,620,800 | Train Loss EWMA: 3.8862 | Learning Rate: 0.002000 | Progress: 0.30310
+Step 3,725 | Tokens: 61,030,400 | Train Loss EWMA: 3.8782 | Learning Rate: 0.002000 | Progress: 0.30515
+Step 3,750 | Tokens: 61,440,000 | Train Loss EWMA: 3.8692 | Learning Rate: 0.002000 | Progress: 0.30720
+Step 3,775 | Tokens: 61,849,600 | Train Loss EWMA: 3.8589 | Learning Rate: 0.002000 | Progress: 0.30925
+Step 3,800 | Tokens: 62,259,200 | Train Loss EWMA: 3.8560 | Learning Rate: 0.002000 | Progress: 0.31130
+Step 3,825 | Tokens: 62,668,800 | Train Loss EWMA: 3.8484 | Learning Rate: 0.002000 | Progress: 0.31334
+Step 3,850 | Tokens: 63,078,400 | Train Loss EWMA: 3.8607 | Learning Rate: 0.002000 | Progress: 0.31539
+Step 3,875 | Tokens: 63,488,000 | Train Loss EWMA: 3.8524 | Learning Rate: 0.002000 | Progress: 0.31744
+Step 3,900 | Tokens: 63,897,600 | Train Loss EWMA: 3.8444 | Learning Rate: 0.002000 | Progress: 0.31949
+Step 3,925 | Tokens: 64,307,200 | Train Loss EWMA: 3.8387 | Learning Rate: 0.002000 | Progress: 0.32154
+Step 3,950 | Tokens: 64,716,800 | Train Loss EWMA: 3.8252 | Learning Rate: 0.002000 | Progress: 0.32358
+Step 3,975 | Tokens: 65,126,400 | Train Loss EWMA: 3.8171 | Learning Rate: 0.002000 | Progress: 0.32563
+Step 4,000 | Tokens: 65,536,000 | Train Loss EWMA: 3.8054 | Learning Rate: 0.002000 | Progress: 0.32768
+Step 4,025 | Tokens: 65,945,600 | Train Loss EWMA: 3.7932 | Learning Rate: 0.002000 | Progress: 0.32973
+Step 4,050 | Tokens: 66,355,200 | Train Loss EWMA: 3.7765 | Learning Rate: 0.002000 | Progress: 0.33178
+Step 4,075 | Tokens: 66,764,800 | Train Loss EWMA: 3.7630 | Learning Rate: 0.002000 | Progress: 0.33382
+Step 4,100 | Tokens: 67,174,400 | Train Loss EWMA: 3.7657 | Learning Rate: 0.002000 | Progress: 0.33587
+Step 4,125 | Tokens: 67,584,000 | Train Loss EWMA: 3.7531 | Learning Rate: 0.002000 | Progress: 0.33792
+Step 4,150 | Tokens: 67,993,600 | Train Loss EWMA: 3.7276 | Learning Rate: 0.002000 | Progress: 0.33997
+Step 4,175 | Tokens: 68,403,200 | Train Loss EWMA: 3.7069 | Learning Rate: 0.002000 | Progress: 0.34202
+Step 4,200 | Tokens: 68,812,800 | Train Loss EWMA: 3.7019 | Learning Rate: 0.002000 | Progress: 0.34406
+Step 4,225 | Tokens: 69,222,400 | Train Loss EWMA: 3.7001 | Learning Rate: 0.002000 | Progress: 0.34611
+Step 4,250 | Tokens: 69,632,000 | Train Loss EWMA: 3.6968 | Learning Rate: 0.002000 | Progress: 0.34816
+Step 4,275 | Tokens: 70,041,600 | Train Loss EWMA: 3.6995 | Learning Rate: 0.002000 | Progress: 0.35021
+Step 4,300 | Tokens: 70,451,200 | Train Loss EWMA: 3.6943 | Learning Rate: 0.002000 | Progress: 0.35226
+Step 4,325 | Tokens: 70,860,800 | Train Loss EWMA: 3.6854 | Learning Rate: 0.002000 | Progress: 0.35430
+Step 4,350 | Tokens: 71,270,400 | Train Loss EWMA: 3.6834 | Learning Rate: 0.002000 | Progress: 0.35635
+Step 4,375 | Tokens: 71,680,000 | Train Loss EWMA: 3.6793 | Learning Rate: 0.002000 | Progress: 0.35840
+Step 4,400 | Tokens: 72,089,600 | Train Loss EWMA: 3.6818 | Learning Rate: 0.002000 | Progress: 0.36045
+Step 4,425 | Tokens: 72,499,200 | Train Loss EWMA: 3.6631 | Learning Rate: 0.002000 | Progress: 0.36250
+Step 4,450 | Tokens: 72,908,800 | Train Loss EWMA: 3.6550 | Learning Rate: 0.002000 | Progress: 0.36454
+Step 4,475 | Tokens: 73,318,400 | Train Loss EWMA: 3.6399 | Learning Rate: 0.002000 | Progress: 0.36659
+Step 4,500 | Tokens: 73,728,000 | Train Loss EWMA: 3.6418 | Learning Rate: 0.002000 | Progress: 0.36864
+Step 4,525 | Tokens: 74,137,600 | Train Loss EWMA: 3.6428 | Learning Rate: 0.002000 | Progress: 0.37069
+Step 4,550 | Tokens: 74,547,200 | Train Loss EWMA: 3.6424 | Learning Rate: 0.002000 | Progress: 0.37274
+Step 4,575 | Tokens: 74,956,800 | Train Loss EWMA: 3.6558 | Learning Rate: 0.002000 | Progress: 0.37478
+Step 4,600 | Tokens: 75,366,400 | Train Loss EWMA: 3.6381 | Learning Rate: 0.002000 | Progress: 0.37683
+Step 4,625 | Tokens: 75,776,000 | Train Loss EWMA: 3.6285 | Learning Rate: 0.002000 | Progress: 0.37888
+Step 4,650 | Tokens: 76,185,600 | Train Loss EWMA: 3.6271 | Learning Rate: 0.002000 | Progress: 0.38093
+Step 4,675 | Tokens: 76,595,200 | Train Loss EWMA: 3.6312 | Learning Rate: 0.002000 | Progress: 0.38298
+Step 4,700 | Tokens: 77,004,800 | Train Loss EWMA: 3.6206 | Learning Rate: 0.002000 | Progress: 0.38502
+Step 4,725 | Tokens: 77,414,400 | Train Loss EWMA: 3.6028 | Learning Rate: 0.002000 | Progress: 0.38707
+Step 4,750 | Tokens: 77,824,000 | Train Loss EWMA: 3.6037 | Learning Rate: 0.002000 | Progress: 0.38912
+Step 4,775 | Tokens: 78,233,600 | Train Loss EWMA: 3.5925 | Learning Rate: 0.002000 | Progress: 0.39117
+Step 4,800 | Tokens: 78,643,200 | Train Loss EWMA: 3.5849 | Learning Rate: 0.002000 | Progress: 0.39322
+Step 4,825 | Tokens: 79,052,800 | Train Loss EWMA: 3.5872 | Learning Rate: 0.002000 | Progress: 0.39526
+Step 4,850 | Tokens: 79,462,400 | Train Loss EWMA: 3.5801 | Learning Rate: 0.002000 | Progress: 0.39731
+Step 4,875 | Tokens: 79,872,000 | Train Loss EWMA: 3.5871 | Learning Rate: 0.002000 | Progress: 0.39936
+Step 4,900 | Tokens: 80,281,600 | Train Loss EWMA: 3.5841 | Learning Rate: 0.002000 | Progress: 0.40141
+Step 4,925 | Tokens: 80,691,200 | Train Loss EWMA: 3.5802 | Learning Rate: 0.002000 | Progress: 0.40346
+Step 4,950 | Tokens: 81,100,800 | Train Loss EWMA: 3.5672 | Learning Rate: 0.002000 | Progress: 0.40550
+Step 4,975 | Tokens: 81,510,400 | Train Loss EWMA: 3.5583 | Learning Rate: 0.002000 | Progress: 0.40755
+Step 5,000 | Tokens: 81,920,000 | Train Loss EWMA: 3.5566 | Learning Rate: 0.002000 | Progress: 0.40960
+Step 5,025 | Tokens: 82,329,600 | Train Loss EWMA: 3.5528 | Learning Rate: 0.002000 | Progress: 0.41165
+Step 5,050 | Tokens: 82,739,200 | Train Loss EWMA: 3.5465 | Learning Rate: 0.002000 | Progress: 0.41370
+Step 5,075 | Tokens: 83,148,800 | Train Loss EWMA: 3.5457 | Learning Rate: 0.002000 | Progress: 0.41574
+Step 5,100 | Tokens: 83,558,400 | Train Loss EWMA: 3.5418 | Learning Rate: 0.002000 | Progress: 0.41779
+Step 5,125 | Tokens: 83,968,000 | Train Loss EWMA: 3.5335 | Learning Rate: 0.002000 | Progress: 0.41984
+Step 5,150 | Tokens: 84,377,600 | Train Loss EWMA: 3.5211 | Learning Rate: 0.002000 | Progress: 0.42189
+Step 5,175 | Tokens: 84,787,200 | Train Loss EWMA: 3.5177 | Learning Rate: 0.002000 | Progress: 0.42394
+Step 5,200 | Tokens: 85,196,800 | Train Loss EWMA: 3.5111 | Learning Rate: 0.002000 | Progress: 0.42598
+Step 5,225 | Tokens: 85,606,400 | Train Loss EWMA: 3.5202 | Learning Rate: 0.002000 | Progress: 0.42803
+Step 5,250 | Tokens: 86,016,000 | Train Loss EWMA: 3.5188 | Learning Rate: 0.002000 | Progress: 0.43008
+Step 5,275 | Tokens: 86,425,600 | Train Loss EWMA: 3.5188 | Learning Rate: 0.002000 | Progress: 0.43213
+Step 5,300 | Tokens: 86,835,200 | Train Loss EWMA: 3.5139 | Learning Rate: 0.002000 | Progress: 0.43418
+Step 5,325 | Tokens: 87,244,800 | Train Loss EWMA: 3.5196 | Learning Rate: 0.002000 | Progress: 0.43622
+Step 5,350 | Tokens: 87,654,400 | Train Loss EWMA: 3.5132 | Learning Rate: 0.002000 | Progress: 0.43827
+Step 5,375 | Tokens: 88,064,000 | Train Loss EWMA: 3.4976 | Learning Rate: 0.002000 | Progress: 0.44032
+Step 5,400 | Tokens: 88,473,600 | Train Loss EWMA: 3.4976 | Learning Rate: 0.002000 | Progress: 0.44237
+Step 5,425 | Tokens: 88,883,200 | Train Loss EWMA: 3.4910 | Learning Rate: 0.002000 | Progress: 0.44442
+Step 5,450 | Tokens: 89,292,800 | Train Loss EWMA: 3.5016 | Learning Rate: 0.002000 | Progress: 0.44646
+Step 5,475 | Tokens: 89,702,400 | Train Loss EWMA: 3.4955 | Learning Rate: 0.002000 | Progress: 0.44851
+Step 5,500 | Tokens: 90,112,000 | Train Loss EWMA: 3.4884 | Learning Rate: 0.002000 | Progress: 0.45056
+Step 5,525 | Tokens: 90,521,600 | Train Loss EWMA: 3.4931 | Learning Rate: 0.002000 | Progress: 0.45261
+Step 5,550 | Tokens: 90,931,200 | Train Loss EWMA: 3.4833 | Learning Rate: 0.002000 | Progress: 0.45466
+Step 5,575 | Tokens: 91,340,800 | Train Loss EWMA: 3.4829 | Learning Rate: 0.002000 | Progress: 0.45670
+Step 5,600 | Tokens: 91,750,400 | Train Loss EWMA: 3.4754 | Learning Rate: 0.002000 | Progress: 0.45875
+Step 5,625 | Tokens: 92,160,000 | Train Loss EWMA: 3.4733 | Learning Rate: 0.002000 | Progress: 0.46080
+Step 5,650 | Tokens: 92,569,600 | Train Loss EWMA: 3.4805 | Learning Rate: 0.002000 | Progress: 0.46285
+Step 5,675 | Tokens: 92,979,200 | Train Loss EWMA: 3.4888 | Learning Rate: 0.002000 | Progress: 0.46490
+Step 5,700 | Tokens: 93,388,800 | Train Loss EWMA: 3.4737 | Learning Rate: 0.002000 | Progress: 0.46694
+Step 5,725 | Tokens: 93,798,400 | Train Loss EWMA: 3.4595 | Learning Rate: 0.002000 | Progress: 0.46899
+Step 5,750 | Tokens: 94,208,000 | Train Loss EWMA: 3.4584 | Learning Rate: 0.002000 | Progress: 0.47104
+Step 5,775 | Tokens: 94,617,600 | Train Loss EWMA: 3.4535 | Learning Rate: 0.002000 | Progress: 0.47309
+Step 5,800 | Tokens: 95,027,200 | Train Loss EWMA: 3.4611 | Learning Rate: 0.002000 | Progress: 0.47514
+Step 5,825 | Tokens: 95,436,800 | Train Loss EWMA: 3.4592 | Learning Rate: 0.002000 | Progress: 0.47718
+Step 5,850 | Tokens: 95,846,400 | Train Loss EWMA: 3.4532 | Learning Rate: 0.002000 | Progress: 0.47923
+Step 5,875 | Tokens: 96,256,000 | Train Loss EWMA: 3.4626 | Learning Rate: 0.002000 | Progress: 0.48128
+Step 5,900 | Tokens: 96,665,600 | Train Loss EWMA: 3.4503 | Learning Rate: 0.002000 | Progress: 0.48333
+Step 5,925 | Tokens: 97,075,200 | Train Loss EWMA: 3.4532 | Learning Rate: 0.002000 | Progress: 0.48538
+Step 5,950 | Tokens: 97,484,800 | Train Loss EWMA: 3.4497 | Learning Rate: 0.002000 | Progress: 0.48742
+Step 5,975 | Tokens: 97,894,400 | Train Loss EWMA: 3.4471 | Learning Rate: 0.002000 | Progress: 0.48947
+Step 6,000 | Tokens: 98,304,000 | Train Loss EWMA: 3.4488 | Learning Rate: 0.002000 | Progress: 0.49152
+Step 6,025 | Tokens: 98,713,600 | Train Loss EWMA: 3.4502 | Learning Rate: 0.002000 | Progress: 0.49357
+Step 6,050 | Tokens: 99,123,200 | Train Loss EWMA: 3.4434 | Learning Rate: 0.002000 | Progress: 0.49562
+Step 6,075 | Tokens: 99,532,800 | Train Loss EWMA: 3.4360 | Learning Rate: 0.002000 | Progress: 0.49766
+Step 6,100 | Tokens: 99,942,400 | Train Loss EWMA: 3.4225 | Learning Rate: 0.002000 | Progress: 0.49971
+Step 6,125 | Tokens: 100,352,000 | Train Loss EWMA: 3.4130 | Learning Rate: 0.002000 | Progress: 0.50176
+Step 6,150 | Tokens: 100,761,600 | Train Loss EWMA: 3.4228 | Learning Rate: 0.002000 | Progress: 0.50381
+Step 6,175 | Tokens: 101,171,200 | Train Loss EWMA: 3.4273 | Learning Rate: 0.002000 | Progress: 0.50586
+Step 6,200 | Tokens: 101,580,800 | Train Loss EWMA: 3.4222 | Learning Rate: 0.002000 | Progress: 0.50790
+Step 6,225 | Tokens: 101,990,400 | Train Loss EWMA: 3.4258 | Learning Rate: 0.002000 | Progress: 0.50995
+Step 6,250 | Tokens: 102,400,000 | Train Loss EWMA: 3.4249 | Learning Rate: 0.002000 | Progress: 0.51200
+Step 6,275 | Tokens: 102,809,600 | Train Loss EWMA: 3.4164 | Learning Rate: 0.002000 | Progress: 0.51405
+Step 6,300 | Tokens: 103,219,200 | Train Loss EWMA: 3.4165 | Learning Rate: 0.002000 | Progress: 0.51610
+Step 6,325 | Tokens: 103,628,800 | Train Loss EWMA: 3.4232 | Learning Rate: 0.002000 | Progress: 0.51814
+Step 6,350 | Tokens: 104,038,400 | Train Loss EWMA: 3.4150 | Learning Rate: 0.002000 | Progress: 0.52019
+Step 6,375 | Tokens: 104,448,000 | Train Loss EWMA: 3.3916 | Learning Rate: 0.002000 | Progress: 0.52224
+Step 6,400 | Tokens: 104,857,600 | Train Loss EWMA: 3.3891 | Learning Rate: 0.002000 | Progress: 0.52429
+Step 6,425 | Tokens: 105,267,200 | Train Loss EWMA: 3.3838 | Learning Rate: 0.002000 | Progress: 0.52634
+Step 6,450 | Tokens: 105,676,800 | Train Loss EWMA: 3.3762 | Learning Rate: 0.002000 | Progress: 0.52838
+Step 6,475 | Tokens: 106,086,400 | Train Loss EWMA: 3.3832 | Learning Rate: 0.002000 | Progress: 0.53043
+Step 6,500 | Tokens: 106,496,000 | Train Loss EWMA: 3.3859 | Learning Rate: 0.002000 | Progress: 0.53248
+Step 6,525 | Tokens: 106,905,600 | Train Loss EWMA: 3.3895 | Learning Rate: 0.002000 | Progress: 0.53453
+Step 6,550 | Tokens: 107,315,200 | Train Loss EWMA: 3.3944 | Learning Rate: 0.002000 | Progress: 0.53658
+Step 6,575 | Tokens: 107,724,800 | Train Loss EWMA: 3.3859 | Learning Rate: 0.002000 | Progress: 0.53862
+Step 6,600 | Tokens: 108,134,400 | Train Loss EWMA: 3.3677 | Learning Rate: 0.002000 | Progress: 0.54067
+Step 6,625 | Tokens: 108,544,000 | Train Loss EWMA: 3.3655 | Learning Rate: 0.002000 | Progress: 0.54272
+Step 6,650 | Tokens: 108,953,600 | Train Loss EWMA: 3.3714 | Learning Rate: 0.002000 | Progress: 0.54477
+Step 6,675 | Tokens: 109,363,200 | Train Loss EWMA: 3.3711 | Learning Rate: 0.002000 | Progress: 0.54682
+Step 6,700 | Tokens: 109,772,800 | Train Loss EWMA: 3.3762 | Learning Rate: 0.002000 | Progress: 0.54886
+Step 6,725 | Tokens: 110,182,400 | Train Loss EWMA: 3.3665 | Learning Rate: 0.002000 | Progress: 0.55091
+Step 6,750 | Tokens: 110,592,000 | Train Loss EWMA: 3.3475 | Learning Rate: 0.002000 | Progress: 0.55296
+Step 6,775 | Tokens: 111,001,600 | Train Loss EWMA: 3.3494 | Learning Rate: 0.002000 | Progress: 0.55501
+Step 6,800 | Tokens: 111,411,200 | Train Loss EWMA: 3.3495 | Learning Rate: 0.002000 | Progress: 0.55706
+Step 6,825 | Tokens: 111,820,800 | Train Loss EWMA: 3.3511 | Learning Rate: 0.002000 | Progress: 0.55910
+Step 6,850 | Tokens: 112,230,400 | Train Loss EWMA: 3.3454 | Learning Rate: 0.002000 | Progress: 0.56115
+Step 6,875 | Tokens: 112,640,000 | Train Loss EWMA: 3.3382 | Learning Rate: 0.002000 | Progress: 0.56320
+Step 6,900 | Tokens: 113,049,600 | Train Loss EWMA: 3.3474 | Learning Rate: 0.002000 | Progress: 0.56525
+Step 6,925 | Tokens: 113,459,200 | Train Loss EWMA: 3.3517 | Learning Rate: 0.002000 | Progress: 0.56730
+Step 6,950 | Tokens: 113,868,800 | Train Loss EWMA: 3.3521 | Learning Rate: 0.002000 | Progress: 0.56934
+Step 6,975 | Tokens: 114,278,400 | Train Loss EWMA: 3.3404 | Learning Rate: 0.002000 | Progress: 0.57139
+Step 7,000 | Tokens: 114,688,000 | Train Loss EWMA: 3.3326 | Learning Rate: 0.002000 | Progress: 0.57344
+Step 7,025 | Tokens: 115,097,600 | Train Loss EWMA: 3.3347 | Learning Rate: 0.002000 | Progress: 0.57549
+Step 7,050 | Tokens: 115,507,200 | Train Loss EWMA: 3.3394 | Learning Rate: 0.002000 | Progress: 0.57754
+Step 7,075 | Tokens: 115,916,800 | Train Loss EWMA: 3.3360 | Learning Rate: 0.002000 | Progress: 0.57958
+Step 7,100 | Tokens: 116,326,400 | Train Loss EWMA: 3.3328 | Learning Rate: 0.002000 | Progress: 0.58163
+Step 7,125 | Tokens: 116,736,000 | Train Loss EWMA: 3.3468 | Learning Rate: 0.002000 | Progress: 0.58368
+Step 7,150 | Tokens: 117,145,600 | Train Loss EWMA: 3.3566 | Learning Rate: 0.002000 | Progress: 0.58573
+Step 7,175 | Tokens: 117,555,200 | Train Loss EWMA: 3.3544 | Learning Rate: 0.002000 | Progress: 0.58778
+Step 7,200 | Tokens: 117,964,800 | Train Loss EWMA: 3.3426 | Learning Rate: 0.002000 | Progress: 0.58982
+Step 7,225 | Tokens: 118,374,400 | Train Loss EWMA: 3.3521 | Learning Rate: 0.002000 | Progress: 0.59187
+Step 7,250 | Tokens: 118,784,000 | Train Loss EWMA: 3.3518 | Learning Rate: 0.002000 | Progress: 0.59392
+Step 7,275 | Tokens: 119,193,600 | Train Loss EWMA: 3.3484 | Learning Rate: 0.002000 | Progress: 0.59597
+Step 7,300 | Tokens: 119,603,200 | Train Loss EWMA: 3.3463 | Learning Rate: 0.002000 | Progress: 0.59802
+Step 7,325 | Tokens: 120,012,800 | Train Loss EWMA: 3.3412 | Learning Rate: 0.002000 | Progress: 0.60006
+Step 7,350 | Tokens: 120,422,400 | Train Loss EWMA: 3.3486 | Learning Rate: 0.002000 | Progress: 0.60211
+Step 7,375 | Tokens: 120,832,000 | Train Loss EWMA: 3.3408 | Learning Rate: 0.002000 | Progress: 0.60416
+Step 7,400 | Tokens: 121,241,600 | Train Loss EWMA: 3.3422 | Learning Rate: 0.002000 | Progress: 0.60621
+Step 7,425 | Tokens: 121,651,200 | Train Loss EWMA: 3.3328 | Learning Rate: 0.002000 | Progress: 0.60826
+Step 7,450 | Tokens: 122,060,800 | Train Loss EWMA: 3.3350 | Learning Rate: 0.002000 | Progress: 0.61030
+Step 7,475 | Tokens: 122,470,400 | Train Loss EWMA: 3.3268 | Learning Rate: 0.002000 | Progress: 0.61235
+Step 7,500 | Tokens: 122,880,000 | Train Loss EWMA: 3.3252 | Learning Rate: 0.002000 | Progress: 0.61440
+Step 7,525 | Tokens: 123,289,600 | Train Loss EWMA: 3.3194 | Learning Rate: 0.002000 | Progress: 0.61645
+Step 7,550 | Tokens: 123,699,200 | Train Loss EWMA: 3.3119 | Learning Rate: 0.002000 | Progress: 0.61850
+Step 7,575 | Tokens: 124,108,800 | Train Loss EWMA: 3.3084 | Learning Rate: 0.002000 | Progress: 0.62054
+Step 7,600 | Tokens: 124,518,400 | Train Loss EWMA: 3.3011 | Learning Rate: 0.002000 | Progress: 0.62259
+Step 7,625 | Tokens: 124,928,000 | Train Loss EWMA: 3.3096 | Learning Rate: 0.002000 | Progress: 0.62464
+Step 7,650 | Tokens: 125,337,600 | Train Loss EWMA: 3.3090 | Learning Rate: 0.002000 | Progress: 0.62669
+Step 7,675 | Tokens: 125,747,200 | Train Loss EWMA: 3.3151 | Learning Rate: 0.002000 | Progress: 0.62874
+Step 7,700 | Tokens: 126,156,800 | Train Loss EWMA: 3.3216 | Learning Rate: 0.002000 | Progress: 0.63078
+Step 7,725 | Tokens: 126,566,400 | Train Loss EWMA: 3.3112 | Learning Rate: 0.002000 | Progress: 0.63283
+Step 7,750 | Tokens: 126,976,000 | Train Loss EWMA: 3.3036 | Learning Rate: 0.002000 | Progress: 0.63488
+Step 7,775 | Tokens: 127,385,600 | Train Loss EWMA: 3.3143 | Learning Rate: 0.002000 | Progress: 0.63693
+Step 7,800 | Tokens: 127,795,200 | Train Loss EWMA: 3.3165 | Learning Rate: 0.002000 | Progress: 0.63898
+Step 7,825 | Tokens: 128,204,800 | Train Loss EWMA: 3.3027 | Learning Rate: 0.002000 | Progress: 0.64102
+Step 7,850 | Tokens: 128,614,400 | Train Loss EWMA: 3.3096 | Learning Rate: 0.002000 | Progress: 0.64307
+Step 7,875 | Tokens: 129,024,000 | Train Loss EWMA: 3.3111 | Learning Rate: 0.002000 | Progress: 0.64512
+Step 7,900 | Tokens: 129,433,600 | Train Loss EWMA: 3.3099 | Learning Rate: 0.002000 | Progress: 0.64717
+Step 7,925 | Tokens: 129,843,200 | Train Loss EWMA: 3.2937 | Learning Rate: 0.002000 | Progress: 0.64922
+Step 7,950 | Tokens: 130,252,800 | Train Loss EWMA: 3.2916 | Learning Rate: 0.002000 | Progress: 0.65126
+Step 7,975 | Tokens: 130,662,400 | Train Loss EWMA: 3.2952 | Learning Rate: 0.002000 | Progress: 0.65331
+Step 8,000 | Tokens: 131,072,000 | Train Loss EWMA: 3.3093 | Learning Rate: 0.002000 | Progress: 0.65536
+Step 8,025 | Tokens: 131,481,600 | Train Loss EWMA: 3.3038 | Learning Rate: 0.002000 | Progress: 0.65741
+Step 8,050 | Tokens: 131,891,200 | Train Loss EWMA: 3.2986 | Learning Rate: 0.002000 | Progress: 0.65946
+Step 8,075 | Tokens: 132,300,800 | Train Loss EWMA: 3.2884 | Learning Rate: 0.002000 | Progress: 0.66150
+Step 8,100 | Tokens: 132,710,400 | Train Loss EWMA: 3.2673 | Learning Rate: 0.002000 | Progress: 0.66355
+Step 8,125 | Tokens: 133,120,000 | Train Loss EWMA: 3.2634 | Learning Rate: 0.002000 | Progress: 0.66560
+Step 8,150 | Tokens: 133,529,600 | Train Loss EWMA: 3.2679 | Learning Rate: 0.002000 | Progress: 0.66765
+Step 8,175 | Tokens: 133,939,200 | Train Loss EWMA: 3.2677 | Learning Rate: 0.002000 | Progress: 0.66970
+Step 8,200 | Tokens: 134,348,800 | Train Loss EWMA: 3.2662 | Learning Rate: 0.002000 | Progress: 0.67174
+Step 8,225 | Tokens: 134,758,400 | Train Loss EWMA: 3.2680 | Learning Rate: 0.002000 | Progress: 0.67379
+Step 8,250 | Tokens: 135,168,000 | Train Loss EWMA: 3.2757 | Learning Rate: 0.002000 | Progress: 0.67584
+Step 8,275 | Tokens: 135,577,600 | Train Loss EWMA: 3.2757 | Learning Rate: 0.002000 | Progress: 0.67789
+Step 8,300 | Tokens: 135,987,200 | Train Loss EWMA: 3.2805 | Learning Rate: 0.002000 | Progress: 0.67994
+Step 8,325 | Tokens: 136,396,800 | Train Loss EWMA: 3.2819 | Learning Rate: 0.002000 | Progress: 0.68198
+Step 8,350 | Tokens: 136,806,400 | Train Loss EWMA: 3.2686 | Learning Rate: 0.002000 | Progress: 0.68403
+Step 8,375 | Tokens: 137,216,000 | Train Loss EWMA: 3.2844 | Learning Rate: 0.002000 | Progress: 0.68608
+Step 8,400 | Tokens: 137,625,600 | Train Loss EWMA: 3.2860 | Learning Rate: 0.002000 | Progress: 0.68813
+Step 8,425 | Tokens: 138,035,200 | Train Loss EWMA: 3.2741 | Learning Rate: 0.002000 | Progress: 0.69018
+Step 8,450 | Tokens: 138,444,800 | Train Loss EWMA: 3.2696 | Learning Rate: 0.002000 | Progress: 0.69222
+Step 8,475 | Tokens: 138,854,400 | Train Loss EWMA: 3.2742 | Learning Rate: 0.002000 | Progress: 0.69427
+Step 8,500 | Tokens: 139,264,000 | Train Loss EWMA: 3.2747 | Learning Rate: 0.002000 | Progress: 0.69632
+Step 8,525 | Tokens: 139,673,600 | Train Loss EWMA: 3.2819 | Learning Rate: 0.002000 | Progress: 0.69837
+Step 8,550 | Tokens: 140,083,200 | Train Loss EWMA: 3.2771 | Learning Rate: 0.002000 | Progress: 0.70042
+Step 8,575 | Tokens: 140,492,800 | Train Loss EWMA: 3.2612 | Learning Rate: 0.002000 | Progress: 0.70246
+Step 8,600 | Tokens: 140,902,400 | Train Loss EWMA: 3.2630 | Learning Rate: 0.002000 | Progress: 0.70451
+Step 8,625 | Tokens: 141,312,000 | Train Loss EWMA: 3.2647 | Learning Rate: 0.002000 | Progress: 0.70656
+Step 8,650 | Tokens: 141,721,600 | Train Loss EWMA: 3.2633 | Learning Rate: 0.002000 | Progress: 0.70861
+Step 8,675 | Tokens: 142,131,200 | Train Loss EWMA: 3.2551 | Learning Rate: 0.002000 | Progress: 0.71066
+Step 8,700 | Tokens: 142,540,800 | Train Loss EWMA: 3.2570 | Learning Rate: 0.002000 | Progress: 0.71270
+Step 8,725 | Tokens: 142,950,400 | Train Loss EWMA: 3.2586 | Learning Rate: 0.002000 | Progress: 0.71475
+Step 8,750 | Tokens: 143,360,000 | Train Loss EWMA: 3.2542 | Learning Rate: 0.002000 | Progress: 0.71680
+Step 8,775 | Tokens: 143,769,600 | Train Loss EWMA: 3.2597 | Learning Rate: 0.002000 | Progress: 0.71885
+Step 8,800 | Tokens: 144,179,200 | Train Loss EWMA: 3.2630 | Learning Rate: 0.002000 | Progress: 0.72090
+Step 8,825 | Tokens: 144,588,800 | Train Loss EWMA: 3.2710 | Learning Rate: 0.002000 | Progress: 0.72294
+Step 8,850 | Tokens: 144,998,400 | Train Loss EWMA: 3.2742 | Learning Rate: 0.002000 | Progress: 0.72499
+Step 8,875 | Tokens: 145,408,000 | Train Loss EWMA: 3.2704 | Learning Rate: 0.002000 | Progress: 0.72704
+Step 8,900 | Tokens: 145,817,600 | Train Loss EWMA: 3.2649 | Learning Rate: 0.002000 | Progress: 0.72909
+Step 8,925 | Tokens: 146,227,200 | Train Loss EWMA: 3.2597 | Learning Rate: 0.002000 | Progress: 0.73114
+Step 8,950 | Tokens: 146,636,800 | Train Loss EWMA: 3.2633 | Learning Rate: 0.002000 | Progress: 0.73318
+Step 8,975 | Tokens: 147,046,400 | Train Loss EWMA: 3.2454 | Learning Rate: 0.002000 | Progress: 0.73523
+Step 9,000 | Tokens: 147,456,000 | Train Loss EWMA: 3.2345 | Learning Rate: 0.002000 | Progress: 0.73728
+Step 9,025 | Tokens: 147,865,600 | Train Loss EWMA: 3.2221 | Learning Rate: 0.002000 | Progress: 0.73933
+Step 9,050 | Tokens: 148,275,200 | Train Loss EWMA: 3.2130 | Learning Rate: 0.002000 | Progress: 0.74138
+Step 9,075 | Tokens: 148,684,800 | Train Loss EWMA: 3.2175 | Learning Rate: 0.002000 | Progress: 0.74342
+Step 9,100 | Tokens: 149,094,400 | Train Loss EWMA: 3.2219 | Learning Rate: 0.002000 | Progress: 0.74547
+Step 9,125 | Tokens: 149,504,000 | Train Loss EWMA: 3.2172 | Learning Rate: 0.002000 | Progress: 0.74752
+Step 9,150 | Tokens: 149,913,600 | Train Loss EWMA: 3.2217 | Learning Rate: 0.002000 | Progress: 0.74957
+Step 9,175 | Tokens: 150,323,200 | Train Loss EWMA: 3.2268 | Learning Rate: 0.002000 | Progress: 0.75162
+Step 9,200 | Tokens: 150,732,800 | Train Loss EWMA: 3.2353 | Learning Rate: 0.002000 | Progress: 0.75366
+Step 9,225 | Tokens: 151,142,400 | Train Loss EWMA: 3.2195 | Learning Rate: 0.002000 | Progress: 0.75571
+Step 9,250 | Tokens: 151,552,000 | Train Loss EWMA: 3.2266 | Learning Rate: 0.002000 | Progress: 0.75776
+Step 9,275 | Tokens: 151,961,600 | Train Loss EWMA: 3.2196 | Learning Rate: 0.002000 | Progress: 0.75981
+Step 9,300 | Tokens: 152,371,200 | Train Loss EWMA: 3.2110 | Learning Rate: 0.002000 | Progress: 0.76186
+Step 9,325 | Tokens: 152,780,800 | Train Loss EWMA: 3.2074 | Learning Rate: 0.002000 | Progress: 0.76390
+Step 9,350 | Tokens: 153,190,400 | Train Loss EWMA: 3.2033 | Learning Rate: 0.002000 | Progress: 0.76595
+Step 9,375 | Tokens: 153,600,000 | Train Loss EWMA: 3.2082 | Learning Rate: 0.002000 | Progress: 0.76800
+Step 9,400 | Tokens: 154,009,600 | Train Loss EWMA: 3.2213 | Learning Rate: 0.002000 | Progress: 0.77005
+Step 9,425 | Tokens: 154,419,200 | Train Loss EWMA: 3.2260 | Learning Rate: 0.002000 | Progress: 0.77210
+Step 9,450 | Tokens: 154,828,800 | Train Loss EWMA: 3.2301 | Learning Rate: 0.002000 | Progress: 0.77414
+Step 9,475 | Tokens: 155,238,400 | Train Loss EWMA: 3.2305 | Learning Rate: 0.002000 | Progress: 0.77619
+Step 9,500 | Tokens: 155,648,000 | Train Loss EWMA: 3.2287 | Learning Rate: 0.002000 | Progress: 0.77824
+Step 9,525 | Tokens: 156,057,600 | Train Loss EWMA: 3.2278 | Learning Rate: 0.002000 | Progress: 0.78029
+Step 9,550 | Tokens: 156,467,200 | Train Loss EWMA: 3.2307 | Learning Rate: 0.002000 | Progress: 0.78234
+Step 9,575 | Tokens: 156,876,800 | Train Loss EWMA: 3.2374 | Learning Rate: 0.002000 | Progress: 0.78438
+Step 9,600 | Tokens: 157,286,400 | Train Loss EWMA: 3.2301 | Learning Rate: 0.002000 | Progress: 0.78643
+Step 9,625 | Tokens: 157,696,000 | Train Loss EWMA: 3.2256 | Learning Rate: 0.002000 | Progress: 0.78848
+Step 9,650 | Tokens: 158,105,600 | Train Loss EWMA: 3.2229 | Learning Rate: 0.002000 | Progress: 0.79053
+Step 9,675 | Tokens: 158,515,200 | Train Loss EWMA: 3.2243 | Learning Rate: 0.002000 | Progress: 0.79258
+Step 9,700 | Tokens: 158,924,800 | Train Loss EWMA: 3.2207 | Learning Rate: 0.002000 | Progress: 0.79462
+Step 9,725 | Tokens: 159,334,400 | Train Loss EWMA: 3.2197 | Learning Rate: 0.002000 | Progress: 0.79667
+Step 9,750 | Tokens: 159,744,000 | Train Loss EWMA: 3.2224 | Learning Rate: 0.002000 | Progress: 0.79872
+Step 9,775 | Tokens: 160,153,600 | Train Loss EWMA: 3.2139 | Learning Rate: 0.002000 | Progress: 0.80077
+Step 9,800 | Tokens: 160,563,200 | Train Loss EWMA: 3.2267 | Learning Rate: 0.002000 | Progress: 0.80282
+Step 9,825 | Tokens: 160,972,800 | Train Loss EWMA: 3.2141 | Learning Rate: 0.002000 | Progress: 0.80486
+Step 9,850 | Tokens: 161,382,400 | Train Loss EWMA: 3.2033 | Learning Rate: 0.002000 | Progress: 0.80691
+Step 9,875 | Tokens: 161,792,000 | Train Loss EWMA: 3.2022 | Learning Rate: 0.002000 | Progress: 0.80896
+Step 9,900 | Tokens: 162,201,600 | Train Loss EWMA: 3.2099 | Learning Rate: 0.002000 | Progress: 0.81101
+Step 9,925 | Tokens: 162,611,200 | Train Loss EWMA: 3.2073 | Learning Rate: 0.002000 | Progress: 0.81306
+Step 9,950 | Tokens: 163,020,800 | Train Loss EWMA: 3.2139 | Learning Rate: 0.002000 | Progress: 0.81510
+Step 9,975 | Tokens: 163,430,400 | Train Loss EWMA: 3.2110 | Learning Rate: 0.002000 | Progress: 0.81715
+Step 10,000 | Tokens: 163,840,000 | Train Loss EWMA: 3.1995 | Learning Rate: 0.002000 | Progress: 0.81920
+Step 10,025 | Tokens: 164,249,600 | Train Loss EWMA: 3.2068 | Learning Rate: 0.002000 | Progress: 0.82125
+Step 10,050 | Tokens: 164,659,200 | Train Loss EWMA: 3.2120 | Learning Rate: 0.002000 | Progress: 0.82330
+Step 10,075 | Tokens: 165,068,800 | Train Loss EWMA: 3.2022 | Learning Rate: 0.002000 | Progress: 0.82534
+Step 10,100 | Tokens: 165,478,400 | Train Loss EWMA: 3.2079 | Learning Rate: 0.002000 | Progress: 0.82739
+Step 10,125 | Tokens: 165,888,000 | Train Loss EWMA: 3.2116 | Learning Rate: 0.002000 | Progress: 0.82944
+Step 10,150 | Tokens: 166,297,600 | Train Loss EWMA: 3.2146 | Learning Rate: 0.002000 | Progress: 0.83149
+Step 10,175 | Tokens: 166,707,200 | Train Loss EWMA: 3.2171 | Learning Rate: 0.002000 | Progress: 0.83354
+Step 10,200 | Tokens: 167,116,800 | Train Loss EWMA: 3.2082 | Learning Rate: 0.002000 | Progress: 0.83558
+Step 10,225 | Tokens: 167,526,400 | Train Loss EWMA: 3.2121 | Learning Rate: 0.002000 | Progress: 0.83763
+Step 10,250 | Tokens: 167,936,000 | Train Loss EWMA: 3.1960 | Learning Rate: 0.002000 | Progress: 0.83968
+Step 10,275 | Tokens: 168,345,600 | Train Loss EWMA: 3.1951 | Learning Rate: 0.002000 | Progress: 0.84173
+Step 10,300 | Tokens: 168,755,200 | Train Loss EWMA: 3.1999 | Learning Rate: 0.002000 | Progress: 0.84378
+Step 10,325 | Tokens: 169,164,800 | Train Loss EWMA: 3.1983 | Learning Rate: 0.002000 | Progress: 0.84582
+Step 10,350 | Tokens: 169,574,400 | Train Loss EWMA: 3.2011 | Learning Rate: 0.002000 | Progress: 0.84787
+Step 10,375 | Tokens: 169,984,000 | Train Loss EWMA: 3.1953 | Learning Rate: 0.002000 | Progress: 0.84992
+Step 10,400 | Tokens: 170,393,600 | Train Loss EWMA: 3.2049 | Learning Rate: 0.002000 | Progress: 0.85197
+Step 10,425 | Tokens: 170,803,200 | Train Loss EWMA: 3.1910 | Learning Rate: 0.002000 | Progress: 0.85402
+Step 10,450 | Tokens: 171,212,800 | Train Loss EWMA: 3.2029 | Learning Rate: 0.002000 | Progress: 0.85606
+Step 10,475 | Tokens: 171,622,400 | Train Loss EWMA: 3.1995 | Learning Rate: 0.002000 | Progress: 0.85811
+Step 10,500 | Tokens: 172,032,000 | Train Loss EWMA: 3.2057 | Learning Rate: 0.002000 | Progress: 0.86016
+Step 10,525 | Tokens: 172,441,600 | Train Loss EWMA: 3.1965 | Learning Rate: 0.002000 | Progress: 0.86221
+Step 10,550 | Tokens: 172,851,200 | Train Loss EWMA: 3.2017 | Learning Rate: 0.002000 | Progress: 0.86426
+Step 10,575 | Tokens: 173,260,800 | Train Loss EWMA: 3.2050 | Learning Rate: 0.002000 | Progress: 0.86630
+Step 10,600 | Tokens: 173,670,400 | Train Loss EWMA: 3.1941 | Learning Rate: 0.002000 | Progress: 0.86835
+Step 10,625 | Tokens: 174,080,000 | Train Loss EWMA: 3.1932 | Learning Rate: 0.002000 | Progress: 0.87040
+Step 10,650 | Tokens: 174,489,600 | Train Loss EWMA: 3.1970 | Learning Rate: 0.002000 | Progress: 0.87245
+Step 10,675 | Tokens: 174,899,200 | Train Loss EWMA: 3.1865 | Learning Rate: 0.002000 | Progress: 0.87450
+Step 10,700 | Tokens: 175,308,800 | Train Loss EWMA: 3.1891 | Learning Rate: 0.002000 | Progress: 0.87654
+Step 10,725 | Tokens: 175,718,400 | Train Loss EWMA: 3.1873 | Learning Rate: 0.002000 | Progress: 0.87859
+Step 10,750 | Tokens: 176,128,000 | Train Loss EWMA: 3.1900 | Learning Rate: 0.002000 | Progress: 0.88064
+Step 10,775 | Tokens: 176,537,600 | Train Loss EWMA: 3.1732 | Learning Rate: 0.002000 | Progress: 0.88269
+Step 10,800 | Tokens: 176,947,200 | Train Loss EWMA: 3.1697 | Learning Rate: 0.002000 | Progress: 0.88474
+Step 10,825 | Tokens: 177,356,800 | Train Loss EWMA: 3.1642 | Learning Rate: 0.002000 | Progress: 0.88678
+Step 10,850 | Tokens: 177,766,400 | Train Loss EWMA: 3.1719 | Learning Rate: 0.002000 | Progress: 0.88883
+Step 10,875 | Tokens: 178,176,000 | Train Loss EWMA: 3.1689 | Learning Rate: 0.002000 | Progress: 0.89088
+Step 10,900 | Tokens: 178,585,600 | Train Loss EWMA: 3.1665 | Learning Rate: 0.002000 | Progress: 0.89293
+Step 10,925 | Tokens: 178,995,200 | Train Loss EWMA: 3.1684 | Learning Rate: 0.002000 | Progress: 0.89498
+Step 10,950 | Tokens: 179,404,800 | Train Loss EWMA: 3.1706 | Learning Rate: 0.002000 | Progress: 0.89702
+Step 10,975 | Tokens: 179,814,400 | Train Loss EWMA: 3.1702 | Learning Rate: 0.002000 | Progress: 0.89907
+Step 11,000 | Tokens: 180,224,000 | Train Loss EWMA: 3.1686 | Learning Rate: 0.002000 | Progress: 0.90112
+Step 11,025 | Tokens: 180,633,600 | Train Loss EWMA: 3.1701 | Learning Rate: 0.002000 | Progress: 0.90317
+Step 11,050 | Tokens: 181,043,200 | Train Loss EWMA: 3.1675 | Learning Rate: 0.002000 | Progress: 0.90522
+Step 11,075 | Tokens: 181,452,800 | Train Loss EWMA: 3.1640 | Learning Rate: 0.002000 | Progress: 0.90726
+Step 11,100 | Tokens: 181,862,400 | Train Loss EWMA: 3.1633 | Learning Rate: 0.002000 | Progress: 0.90931
+Step 11,125 | Tokens: 182,272,000 | Train Loss EWMA: 3.1685 | Learning Rate: 0.002000 | Progress: 0.91136
+Step 11,150 | Tokens: 182,681,600 | Train Loss EWMA: 3.1668 | Learning Rate: 0.002000 | Progress: 0.91341
+Step 11,175 | Tokens: 183,091,200 | Train Loss EWMA: 3.1686 | Learning Rate: 0.002000 | Progress: 0.91546
+Step 11,200 | Tokens: 183,500,800 | Train Loss EWMA: 3.1714 | Learning Rate: 0.002000 | Progress: 0.91750
+Step 11,225 | Tokens: 183,910,400 | Train Loss EWMA: 3.1597 | Learning Rate: 0.002000 | Progress: 0.91955
+Step 11,250 | Tokens: 184,320,000 | Train Loss EWMA: 3.1599 | Learning Rate: 0.002000 | Progress: 0.92160
+Step 11,275 | Tokens: 184,729,600 | Train Loss EWMA: 3.1580 | Learning Rate: 0.002000 | Progress: 0.92365
+Step 11,300 | Tokens: 185,139,200 | Train Loss EWMA: 3.1546 | Learning Rate: 0.002000 | Progress: 0.92570
+Step 11,325 | Tokens: 185,548,800 | Train Loss EWMA: 3.1514 | Learning Rate: 0.002000 | Progress: 0.92774
+Step 11,350 | Tokens: 185,958,400 | Train Loss EWMA: 3.1546 | Learning Rate: 0.002000 | Progress: 0.92979
+Step 11,375 | Tokens: 186,368,000 | Train Loss EWMA: 3.1505 | Learning Rate: 0.002000 | Progress: 0.93184
+Step 11,400 | Tokens: 186,777,600 | Train Loss EWMA: 3.1569 | Learning Rate: 0.002000 | Progress: 0.93389
+Step 11,425 | Tokens: 187,187,200 | Train Loss EWMA: 3.1561 | Learning Rate: 0.002000 | Progress: 0.93594
+Step 11,450 | Tokens: 187,596,800 | Train Loss EWMA: 3.1543 | Learning Rate: 0.002000 | Progress: 0.93798
+Step 11,475 | Tokens: 188,006,400 | Train Loss EWMA: 3.1627 | Learning Rate: 0.002000 | Progress: 0.94003
+Step 11,500 | Tokens: 188,416,000 | Train Loss EWMA: 3.1612 | Learning Rate: 0.002000 | Progress: 0.94208
+Step 11,525 | Tokens: 188,825,600 | Train Loss EWMA: 3.1575 | Learning Rate: 0.002000 | Progress: 0.94413
+Step 11,550 | Tokens: 189,235,200 | Train Loss EWMA: 3.1458 | Learning Rate: 0.002000 | Progress: 0.94618
+Step 11,575 | Tokens: 189,644,800 | Train Loss EWMA: 3.1480 | Learning Rate: 0.002000 | Progress: 0.94822
+Step 11,600 | Tokens: 190,054,400 | Train Loss EWMA: 3.1406 | Learning Rate: 0.002000 | Progress: 0.95027
+Step 11,625 | Tokens: 190,464,000 | Train Loss EWMA: 3.1354 | Learning Rate: 0.002000 | Progress: 0.95232
+Step 11,650 | Tokens: 190,873,600 | Train Loss EWMA: 3.1445 | Learning Rate: 0.002000 | Progress: 0.95437
+Step 11,675 | Tokens: 191,283,200 | Train Loss EWMA: 3.1448 | Learning Rate: 0.002000 | Progress: 0.95642
+Step 11,700 | Tokens: 191,692,800 | Train Loss EWMA: 3.1522 | Learning Rate: 0.002000 | Progress: 0.95846
+Step 11,725 | Tokens: 192,102,400 | Train Loss EWMA: 3.1450 | Learning Rate: 0.002000 | Progress: 0.96051
+Step 11,750 | Tokens: 192,512,000 | Train Loss EWMA: 3.1446 | Learning Rate: 0.002000 | Progress: 0.96256
+Step 11,775 | Tokens: 192,921,600 | Train Loss EWMA: 3.1456 | Learning Rate: 0.002000 | Progress: 0.96461
+Step 11,800 | Tokens: 193,331,200 | Train Loss EWMA: 3.1429 | Learning Rate: 0.002000 | Progress: 0.96666
+Step 11,825 | Tokens: 193,740,800 | Train Loss EWMA: 3.1465 | Learning Rate: 0.002000 | Progress: 0.96870
+Step 11,850 | Tokens: 194,150,400 | Train Loss EWMA: 3.1536 | Learning Rate: 0.002000 | Progress: 0.97075
+Step 11,875 | Tokens: 194,560,000 | Train Loss EWMA: 3.1504 | Learning Rate: 0.002000 | Progress: 0.97280
+Step 11,900 | Tokens: 194,969,600 | Train Loss EWMA: 3.1592 | Learning Rate: 0.002000 | Progress: 0.97485
+Step 11,925 | Tokens: 195,379,200 | Train Loss EWMA: 3.1665 | Learning Rate: 0.002000 | Progress: 0.97690
+Step 11,950 | Tokens: 195,788,800 | Train Loss EWMA: 3.1527 | Learning Rate: 0.002000 | Progress: 0.97894
+Step 11,975 | Tokens: 196,198,400 | Train Loss EWMA: 3.1323 | Learning Rate: 0.002000 | Progress: 0.98099
+Step 12,000 | Tokens: 196,608,000 | Train Loss EWMA: 3.1383 | Learning Rate: 0.002000 | Progress: 0.98304
+Step 12,025 | Tokens: 197,017,600 | Train Loss EWMA: 3.1446 | Learning Rate: 0.002000 | Progress: 0.98509
+Step 12,050 | Tokens: 197,427,200 | Train Loss EWMA: 3.1493 | Learning Rate: 0.002000 | Progress: 0.98714
+Step 12,075 | Tokens: 197,836,800 | Train Loss EWMA: 3.1435 | Learning Rate: 0.002000 | Progress: 0.98918
+Step 12,100 | Tokens: 198,246,400 | Train Loss EWMA: 3.1346 | Learning Rate: 0.002000 | Progress: 0.99123
+Step 12,125 | Tokens: 198,656,000 | Train Loss EWMA: 3.1263 | Learning Rate: 0.002000 | Progress: 0.99328
+Step 12,150 | Tokens: 199,065,600 | Train Loss EWMA: 3.1348 | Learning Rate: 0.002000 | Progress: 0.99533
+Step 12,175 | Tokens: 199,475,200 | Train Loss EWMA: 3.1443 | Learning Rate: 0.002000 | Progress: 0.99738
+Step 12,200 | Tokens: 199,884,800 | Train Loss EWMA: 3.1266 | Learning Rate: 0.002000 | Progress: 0.99942
diff --git a/wandb/run-20260319_091054-lisp43b6/files/requirements.txt b/wandb/run-20260319_091054-lisp43b6/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c32285d10ba18c2e783ff2ead305d5976caef668
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/files/requirements.txt
@@ -0,0 +1,222 @@
+fsspec==2025.3.0
+PyYAML==6.0.2
+certifi==2025.8.3
+comm==0.2.3
+widgetsnbextension==4.0.14
+Jinja2==3.1.6
+rich==14.1.0
+circuitsvis==1.43.3
+hf-xet==1.1.9
+param==2.2.1
+httpcore==1.0.9
+nvidia-cuda-cupti-cu12==12.8.90
+nvidia-cuda-nvrtc-cu12==12.8.93
+asttokens==3.0.0
+filelock==3.19.1
+types-python-dateutil==2.9.0.20250822
+cycler==0.12.1
+stack-data==0.6.3
+jupyter_server==2.17.0
+aiosignal==1.4.0
+xyzservices==2025.4.0
+lark==1.2.2
+ptyprocess==0.7.0
+xxhash==3.5.0
+mpmath==1.3.0
+seaborn==0.13.2
+wadler_lindig==0.1.7
+nbformat==5.10.4
+panel==1.8.0
+accelerate==1.10.1
+plotly==6.3.0
+narwhals==2.4.0
+huggingface-hub==0.34.4
+sentencepiece==0.2.1
+torchvision==0.23.0
+ipython==9.5.0
+tqdm==4.67.1
+contourpy==1.3.3
+nvidia-nvtx-cu12==12.8.90
+nvidia-cuda-runtime-cu12==12.8.90
+yarl==1.20.1
+charset-normalizer==3.4.3
+jupyter-events==0.12.0
+nbclient==0.10.2
+numpy==1.26.4
+decorator==5.2.1
+threadpoolctl==3.6.0
+networkx==3.5
+smmap==5.0.2
+nbconvert==7.16.6
+pytz==2025.2
+aiohappyeyeballs==2.6.1
+requests==2.32.5
+tinycss2==1.4.0
+defusedxml==0.7.1
+matplotlib-inline==0.1.7
+rpds-py==0.27.1
+wandb==0.21.4
+jedi==0.19.2
+pathspec==0.12.1
+transformer-lens==2.16.1
+sympy==1.14.0
+jupyterlab_pygments==0.3.0
+overrides==7.7.0
+notebook_shim==0.2.4
+jupyter==1.1.1
+protobuf==6.32.1
+better-abc==0.0.3
+jsonpointer==3.0.0
+terminado==0.18.1
+cfgv==3.4.0
+rfc3987-syntax==1.1.0
+annotated-types==0.7.0
+pyarrow==21.0.0
+webencodings==0.5.1
+wcwidth==0.2.13
+jupyterlab_server==2.27.3
+argon2-cffi-bindings==25.1.0
+nvidia-nvjitlink-cu12==12.8.93
+jaxtyping==0.3.2
+Pygments==2.19.2
+torch==2.8.0
+rfc3339-validator==0.1.4
+urllib3==2.5.0
+jupyterlab_widgets==3.0.15
+ipykernel==6.30.1
+nvidia-cudnn-cu12==9.10.2.21
+beautifulsoup4==4.13.5
+babel==2.17.0
+pure_eval==0.2.3
+pyparsing==3.2.3
+nvidia-cublas-cu12==12.8.4.1
+regex==2025.9.1
+pycparser==2.23
+soupsieve==2.8
+pytest-cov==7.0.0
+sniffio==1.3.1
+mypy==1.18.1
+notebook==7.4.5
+packaging==25.0
+h11==0.16.0
+psutil==7.0.0
+pexpect==4.9.0
+zstandard==0.25.0
+gitdb==4.0.12
+rfc3986-validator==0.1.1
+pyzmq==27.1.0
+jupyterlab==4.4.7
+toy_models==0.1.0
+torchaudio==2.8.0
+cffi==2.0.0
+mypy_extensions==1.1.0
+attrs==25.3.0
+statsmodels==0.14.6
+transformers==4.56.1
+jupyter_core==5.8.1
+bleach==6.2.0
+fqdn==1.5.1
+async-lru==2.0.5
+nvidia-nccl-cu12==2.27.3
+GitPython==3.1.45
+referencing==0.36.2
+click==8.2.1
+prometheus_client==0.22.1
+bokeh==3.8.0
+httpx==0.28.1
+setuptools==80.9.0
+argon2-cffi==25.1.0
+patsy==1.0.2
+multidict==6.6.4
+pyviz_comms==3.0.6
+arrow==1.3.0
+scikit-learn==1.8.0
+beartype==0.14.1
+ipywidgets==8.1.7
+pydantic_core==2.33.2
+markdown-it-py==4.0.0
+pandas==2.3.2
+virtualenv==20.34.0
+python-dotenv==1.1.1
+isoduration==20.11.0
+python-dateutil==2.9.0.post0
+nodeenv==1.9.1
+nvidia-curand-cu12==10.3.9.90
+webcolors==24.11.1
+MarkupSafe==3.0.2
+nvidia-cusolver-cu12==11.7.3.90
+Send2Trash==1.8.3
+coverage==7.10.6
+jupyter_server_terminals==0.5.3
+debugpy==1.8.16
+json5==0.12.1
+linkify-it-py==2.0.3
+importlib_metadata==8.7.0
+nvidia-cufft-cu12==11.3.3.83
+distlib==0.4.0
+typing-inspection==0.4.1
+identify==2.6.14
+nvidia-cufile-cu12==1.13.1.3
+scipy==1.17.0
+mdurl==0.1.2
+websocket-client==1.8.0
+jsonschema==4.25.1
+python-json-logger==3.3.0
+typing_extensions==4.15.0
+tokenizers==0.22.0
+ipympl==0.9.7
+einops==0.8.1
+jupyter_client==8.6.3
+ipython_pygments_lexers==1.1.1
+h5py==3.14.0
+tabulate==0.9.0
+propcache==0.3.2
+ruff==0.13.0
+tornado==6.5.2
+typeguard==4.4.4
+tomlkit==0.13.2
+pluggy==1.6.0
+pydantic==2.11.7
+zipp==3.23.0
+fancy-einsum==0.0.3
+fastjsonschema==2.21.2
+datasets==4.0.0
+fonttools==4.59.2
+executing==2.2.1
+pillow==11.3.0
+uc-micro-py==1.0.3
+Markdown==3.9
+pre_commit==4.3.0
+aiohttp==3.12.15
+mistune==3.1.4
+tzdata==2025.2
+parso==0.8.5
+triton==3.4.0
+kiwisolver==1.4.9
+idna==3.10
+multiprocess==0.70.16
+dill==0.3.8
+jupyter-lsp==2.3.0
+platformdirs==4.4.0
+sentry-sdk==2.37.1
+prompt_toolkit==3.0.52
+jsonschema-specifications==2025.9.1
+pytest==8.4.2
+mdit-py-plugins==0.5.0
+transformers-stream-generator==0.0.5
+nvidia-cusparselt-cu12==0.7.1
+joblib==1.5.3
+pandocfilters==1.5.1
+jupyter-console==6.6.3
+anyio==4.10.0
+six==1.17.0
+holoviews==1.21.0
+matplotlib==3.10.6
+colorcet==3.1.0
+uri-template==1.3.0
+nest-asyncio==1.6.0
+nvidia-cusparse-cu12==12.5.8.93
+iniconfig==2.1.0
+traitlets==5.14.3
+safetensors==0.6.2
+frozenlist==1.7.0
diff --git a/wandb/run-20260319_091054-lisp43b6/files/wandb-metadata.json b/wandb/run-20260319_091054-lisp43b6/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..8cdaccbb813cc1f3916cb15b6960d861ebe000ba
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/files/wandb-metadata.json
@@ -0,0 +1,38 @@
+{
+  "os":  "Linux-5.19.0-45-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.7",
+  "startedAt":  "2026-03-19T09:10:54.300170Z",
+  "program":  "<python with no main file>",
+  "git":  {
+    "remote":  "https://github.com/jgroh3/toy_models.git",
+    "commit":  "d722bb952956265d0387df9c35a76703a66824ec"
+  },
+  "email":  "tzfof8@gmail.com",
+  "root":  "/notebooks/toy_models/model_training/model",
+  "host":  "nmvyvu5qhg",
+  "executable":  "/notebooks/toy_models/.toy_models_env/bin/python",
+  "cpu_count":  8,
+  "cpu_count_logical":  8,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "262240792576",
+      "used":  "152260071424"
+    }
+  },
+  "memory":  {
+    "total":  "47332843520"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-ec83d6f5-8082-3e0c-f60c-d8e819df4844"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "jdofomlxripi823w2v6uwgggyst0ywfm"
+}
\ No newline at end of file
diff --git a/wandb/run-20260319_091054-lisp43b6/files/wandb-summary.json b/wandb/run-20260319_091054-lisp43b6/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..d648d4b5668cd036907a2ebafc5a76013eb86cc8
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/files/wandb-summary.json
@@ -0,0 +1 @@
+{"train_loss_ewma":3.1265936626997544,"progress":0.999424,"_step":12200,"_timestamp":1.773921955204926e+09,"tokens_seen":199884800,"train_loss":3.054595470428467,"tokens_per_second":16384,"learning_rate":0.002,"step":12200,"_wandb":{"runtime":10507},"_runtime":10507.186489464}
\ No newline at end of file
diff --git a/wandb/run-20260319_091054-lisp43b6/logs/debug-core.log b/wandb/run-20260319_091054-lisp43b6/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..0a39ffaf87c72097d951c73f05b9843bedbe55fb
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/logs/debug-core.log
@@ -0,0 +1,16 @@
+{"time":"2026-03-19T09:10:54.324476393Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpr62wsped/port-678.txt","pid":678,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-03-19T09:10:54.325154489Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":678}
+{"time":"2026-03-19T09:10:54.325110839Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-678-714-2465202564/socket","Net":"unix"}}
+{"time":"2026-03-19T09:10:54.512560962Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-03-19T09:10:54.517992243Z","level":"INFO","msg":"handleInformInit: received","streamId":"lisp43b6","id":"1(@)"}
+{"time":"2026-03-19T09:10:54.728800254Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"lisp43b6","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.888401158Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"lisp43b6","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891144166Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"lisp43b6","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891165086Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891186306Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891198362Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891236552Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-03-19T12:06:12.891243646Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891306248Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-03-19T12:06:12.891406791Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-678-714-2465202564/socket","Net":"unix"}}
+{"time":"2026-03-19T12:06:12.891450683Z","level":"INFO","msg":"server is closed"}
diff --git a/wandb/run-20260319_091054-lisp43b6/logs/debug-internal.log b/wandb/run-20260319_091054-lisp43b6/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..acf4434179822f6b9a52fc76ffbe4efd3446278f
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/logs/debug-internal.log
@@ -0,0 +1,12 @@
+{"time":"2026-03-19T09:10:54.518147627Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2026-03-19T09:10:54.728754949Z","level":"INFO","msg":"stream: created new stream","id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.728795071Z","level":"INFO","msg":"stream: started","id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.729757806Z","level":"INFO","msg":"sender: started","stream_id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.72979291Z","level":"INFO","msg":"handler: started","stream_id":"lisp43b6"}
+{"time":"2026-03-19T09:10:54.729814605Z","level":"INFO","msg":"writer: started","stream_id":"lisp43b6"}
+{"time":"2026-03-19T12:06:02.518903145Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading data","runtime_seconds":9.420394008},{"desc":"updating run metadata","runtime_seconds":0.001821517}],"total_operations":2}}
+{"time":"2026-03-19T12:06:03.072131148Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-03-19T12:06:12.889325955Z","level":"INFO","msg":"stream: closing","id":"lisp43b6"}
+{"time":"2026-03-19T12:06:12.889384907Z","level":"INFO","msg":"handler: closed","stream_id":"lisp43b6"}
+{"time":"2026-03-19T12:06:12.889456031Z","level":"INFO","msg":"sender: closed","stream_id":"lisp43b6"}
+{"time":"2026-03-19T12:06:12.889466313Z","level":"INFO","msg":"stream: closed","id":"lisp43b6"}
diff --git a/wandb/run-20260319_091054-lisp43b6/logs/debug.log b/wandb/run-20260319_091054-lisp43b6/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..6341541a524f3bf7b2378d00f86247eec82d17aa
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/logs/debug.log
@@ -0,0 +1,26 @@
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Configure stats pid to 678
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/model/wandb/settings
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_091054-lisp43b6/logs/debug.log
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/model/wandb/run-20260319_091054-lisp43b6/logs/debug-internal.log
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:init():813] calling init triggers
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'pile_llama_replace_17367_L8_1024', 'n_layers': 12, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 32000, 'dataset_name': 'eoinf/PL_Replace17367_L2_alldataset', 'tokenizer_name': '', 'seed': 10, 'data_seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 16, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.002, 'lr_vector': 0.001, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 16, 'tokens_per_step': 16384, 'warmup_steps': 1831, 'max_steps': 12207, '_wandb': {}}
+2026-03-19 09:10:54,307 INFO    MainThread:678 [wandb_init.py:init():854] starting backend
+2026-03-19 09:10:54,512 INFO    MainThread:678 [wandb_init.py:init():857] sending inform_init request
+2026-03-19 09:10:54,516 INFO    MainThread:678 [wandb_init.py:init():865] backend started and connected
+2026-03-19 09:10:54,517 INFO    MainThread:678 [wandb_init.py:init():936] updated telemetry
+2026-03-19 09:10:54,557 INFO    MainThread:678 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2026-03-19 09:10:55,330 INFO    MainThread:678 [wandb_init.py:init():1011] starting run threads in backend
+2026-03-19 09:10:55,439 INFO    MainThread:678 [wandb_run.py:_console_start():2506] atexit reg
+2026-03-19 09:10:55,440 INFO    MainThread:678 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2026-03-19 09:10:55,440 INFO    MainThread:678 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2026-03-19 09:10:55,440 INFO    MainThread:678 [wandb_run.py:_redirect():2446] Redirects installed.
+2026-03-19 09:10:55,442 INFO    MainThread:678 [wandb_init.py:init():1049] run started, returning control to user process
+2026-03-19 12:06:02,510 INFO    MainThread:678 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/lisp43b6
+2026-03-19 12:06:02,516 INFO    MainThread:678 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0
+2026-03-19 12:06:02,517 INFO    MainThread:678 [wandb_run.py:_restore():2453] restore
+2026-03-19 12:06:02,517 INFO    MainThread:678 [wandb_run.py:_restore():2459] restore done
+2026-03-19 12:06:12,886 INFO    MainThread:678 [wandb_run.py:_footer_sync_info():3867] logging synced files
diff --git a/wandb/run-20260319_091054-lisp43b6/run-lisp43b6.wandb b/wandb/run-20260319_091054-lisp43b6/run-lisp43b6.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..9b03434644552b1e0b7aaa349fadb13ad80483e3
--- /dev/null
+++ b/wandb/run-20260319_091054-lisp43b6/run-lisp43b6.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a39bc22ec6e2807f72eb2614ed3a666086e32aa82a1fe702f10393b2eaa6c216
+size 8083898