diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..af2a60af65f9b3f795767298954782b88b1f8e4b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250926_233554-wxnxan9r/run-wxnxan9r.wandb filter=lfs diff=lfs merge=lfs -text
diff --git a/checkpoints/metadata_000000032768.json b/checkpoints/metadata_000000032768.json
new file mode 100644
index 0000000000000000000000000000000000000000..4e5a6259a2047d28ba672f861e6f9ed6f92fdf88
--- /dev/null
+++ b/checkpoints/metadata_000000032768.json
@@ -0,0 +1 @@
+{"step": 1, "tokens_seen": 32768, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.869385719299316}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000327680.json b/checkpoints/metadata_000000327680.json
new file mode 100644
index 0000000000000000000000000000000000000000..988966a9b934568c7fd70cc3cbc717fffb968464
--- /dev/null
+++ b/checkpoints/metadata_000000327680.json
@@ -0,0 +1 @@
+{"step": 10, "tokens_seen": 327680, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.863942376726378}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000360448.json b/checkpoints/metadata_000000360448.json
new file mode 100644
index 0000000000000000000000000000000000000000..a087eb88f850726dbdfcc88d7fa9ee69fb811edf
--- /dev/null
+++ b/checkpoints/metadata_000000360448.json
@@ -0,0 +1 @@
+{"step": 11, "tokens_seen": 360448, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.861979302599007}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000425984.json b/checkpoints/metadata_000000425984.json
new file mode 100644
index 0000000000000000000000000000000000000000..206d94d298e0423171cb0421afe1da8416d29c30
--- /dev/null
+++ b/checkpoints/metadata_000000425984.json
@@ -0,0 +1 @@
+{"step": 13, "tokens_seen": 425984, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.856171463869986}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000458752.json b/checkpoints/metadata_000000458752.json
new file mode 100644
index 0000000000000000000000000000000000000000..4edb5dcc05b8cd95fbfbd1caab538e7b884212c7
--- /dev/null
+++ b/checkpoints/metadata_000000458752.json
@@ -0,0 +1 @@
+{"step": 14, "tokens_seen": 458752, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.852995462808556}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000491520.json b/checkpoints/metadata_000000491520.json
new file mode 100644
index 0000000000000000000000000000000000000000..9429f6d8aaadb450d32a8ddcc0f78a6148f6a8d2
--- /dev/null
+++ b/checkpoints/metadata_000000491520.json
@@ -0,0 +1 @@
+{"step": 15, "tokens_seen": 491520, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.849087166505665}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000557056.json b/checkpoints/metadata_000000557056.json
new file mode 100644
index 0000000000000000000000000000000000000000..dca81248d9b14056b34a8a4ab944319164396e44
--- /dev/null
+++ b/checkpoints/metadata_000000557056.json
@@ -0,0 +1 @@
+{"step": 17, "tokens_seen": 557056, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.840072454408835}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000622592.json b/checkpoints/metadata_000000622592.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3601a23fe58b54efb56e7536f9e5c9f15c37e70
--- /dev/null
+++ b/checkpoints/metadata_000000622592.json
@@ -0,0 +1 @@
+{"step": 19, "tokens_seen": 622592, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.829971782833585}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000688128.json b/checkpoints/metadata_000000688128.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ec60c016fd1fa7ab84b35f86df117b45de89e90
--- /dev/null
+++ b/checkpoints/metadata_000000688128.json
@@ -0,0 +1 @@
+{"step": 21, "tokens_seen": 688128, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.817913977491944}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000753664.json b/checkpoints/metadata_000000753664.json
new file mode 100644
index 0000000000000000000000000000000000000000..44552c26276a928328449366ead10697a0eea46f
--- /dev/null
+++ b/checkpoints/metadata_000000753664.json
@@ -0,0 +1 @@
+{"step": 23, "tokens_seen": 753664, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.80478192226603}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000819200.json b/checkpoints/metadata_000000819200.json
new file mode 100644
index 0000000000000000000000000000000000000000..acf80aea1a2c1bc4aed490341bbad1ecbc91ca53
--- /dev/null
+++ b/checkpoints/metadata_000000819200.json
@@ -0,0 +1 @@
+{"step": 25, "tokens_seen": 819200, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.79108913893173}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000917504.json b/checkpoints/metadata_000000917504.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d2292cdacfa90df732b07dfab8410aefa264d53
--- /dev/null
+++ b/checkpoints/metadata_000000917504.json
@@ -0,0 +1 @@
+{"step": 28, "tokens_seen": 917504, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.769061587856811}
\ No newline at end of file
diff --git a/checkpoints/metadata_000000983040.json b/checkpoints/metadata_000000983040.json
new file mode 100644
index 0000000000000000000000000000000000000000..fafa64bb191d7351a9079f11fb423e86146996bb
--- /dev/null
+++ b/checkpoints/metadata_000000983040.json
@@ -0,0 +1 @@
+{"step": 30, "tokens_seen": 983040, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.75334692118638}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001114112.json b/checkpoints/metadata_000001114112.json
new file mode 100644
index 0000000000000000000000000000000000000000..880d458daa76bca1d4893ff4736aa33696c5f3cc
--- /dev/null
+++ b/checkpoints/metadata_000001114112.json
@@ -0,0 +1 @@
+{"step": 34, "tokens_seen": 1114112, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.720532747106743}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001212416.json b/checkpoints/metadata_000001212416.json
new file mode 100644
index 0000000000000000000000000000000000000000..2100039c7443af4eeefc79d88145af2474e7c813
--- /dev/null
+++ b/checkpoints/metadata_000001212416.json
@@ -0,0 +1 @@
+{"step": 37, "tokens_seen": 1212416, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.693520965031334}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001343488.json b/checkpoints/metadata_000001343488.json
new file mode 100644
index 0000000000000000000000000000000000000000..f0400509b61713d68269a6bf35eb5834b7d055b9
--- /dev/null
+++ b/checkpoints/metadata_000001343488.json
@@ -0,0 +1 @@
+{"step": 41, "tokens_seen": 1343488, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.656341522357172}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001474560.json b/checkpoints/metadata_000001474560.json
new file mode 100644
index 0000000000000000000000000000000000000000..42a6f0c968d80bfbf59d51ff0d280c0cb19ea5c8
--- /dev/null
+++ b/checkpoints/metadata_000001474560.json
@@ -0,0 +1 @@
+{"step": 45, "tokens_seen": 1474560, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.616007213533805}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001605632.json b/checkpoints/metadata_000001605632.json
new file mode 100644
index 0000000000000000000000000000000000000000..54bc75e038d42ad98f1f8a50beb0df0f02f9565c
--- /dev/null
+++ b/checkpoints/metadata_000001605632.json
@@ -0,0 +1 @@
+{"step": 49, "tokens_seen": 1605632, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.571887433295842}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001769472.json b/checkpoints/metadata_000001769472.json
new file mode 100644
index 0000000000000000000000000000000000000000..efced1b6d4ab1d8098c4a6b680a8501ec50f0f1a
--- /dev/null
+++ b/checkpoints/metadata_000001769472.json
@@ -0,0 +1 @@
+{"step": 54, "tokens_seen": 1769472, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.512128426068239}
\ No newline at end of file
diff --git a/checkpoints/metadata_000001966080.json b/checkpoints/metadata_000001966080.json
new file mode 100644
index 0000000000000000000000000000000000000000..1cbb0f649e3b2e3b018cd67f40d2fc8c7f0d85bd
--- /dev/null
+++ b/checkpoints/metadata_000001966080.json
@@ -0,0 +1 @@
+{"step": 60, "tokens_seen": 1966080, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.432821735057082}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002162688.json b/checkpoints/metadata_000002162688.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d019bbd133e73238feae3ef3c9a9b58345518ee
--- /dev/null
+++ b/checkpoints/metadata_000002162688.json
@@ -0,0 +1 @@
+{"step": 66, "tokens_seen": 2162688, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.346376037499075}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002359296.json b/checkpoints/metadata_000002359296.json
new file mode 100644
index 0000000000000000000000000000000000000000..6a05382f4a7e5684c89a2e689afa8865a59d6dad
--- /dev/null
+++ b/checkpoints/metadata_000002359296.json
@@ -0,0 +1 @@
+{"step": 72, "tokens_seen": 2359296, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.251224262006552}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002621440.json b/checkpoints/metadata_000002621440.json
new file mode 100644
index 0000000000000000000000000000000000000000..d8689ebaeb3cc57f275e55640cca5c3226c619b8
--- /dev/null
+++ b/checkpoints/metadata_000002621440.json
@@ -0,0 +1 @@
+{"step": 80, "tokens_seen": 2621440, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 10.113908712994371}
\ No newline at end of file
diff --git a/checkpoints/metadata_000002883584.json b/checkpoints/metadata_000002883584.json
new file mode 100644
index 0000000000000000000000000000000000000000..7e77edad1440d6670841758a231f77a86451351e
--- /dev/null
+++ b/checkpoints/metadata_000002883584.json
@@ -0,0 +1 @@
+{"step": 88, "tokens_seen": 2883584, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.967051648617549}
\ No newline at end of file
diff --git a/checkpoints/metadata_000003178496.json b/checkpoints/metadata_000003178496.json
new file mode 100644
index 0000000000000000000000000000000000000000..e154d01c19572b59e4c7d9dbdf61d81db04cb9c3
--- /dev/null
+++ b/checkpoints/metadata_000003178496.json
@@ -0,0 +1 @@
+{"step": 97, "tokens_seen": 3178496, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.791988261583532}
\ No newline at end of file
diff --git a/checkpoints/metadata_000003473408.json b/checkpoints/metadata_000003473408.json
new file mode 100644
index 0000000000000000000000000000000000000000..ae82c964510243efcfdba25e995bd5349e9614b1
--- /dev/null
+++ b/checkpoints/metadata_000003473408.json
@@ -0,0 +1 @@
+{"step": 106, "tokens_seen": 3473408, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.616236149437459}
\ No newline at end of file
diff --git a/checkpoints/metadata_000003833856.json b/checkpoints/metadata_000003833856.json
new file mode 100644
index 0000000000000000000000000000000000000000..d0cada8dbb09715692e46c942071a6ae1e84a34f
--- /dev/null
+++ b/checkpoints/metadata_000003833856.json
@@ -0,0 +1 @@
+{"step": 117, "tokens_seen": 3833856, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.408948975220248}
\ No newline at end of file
diff --git a/checkpoints/metadata_000004227072.json b/checkpoints/metadata_000004227072.json
new file mode 100644
index 0000000000000000000000000000000000000000..4fd8c65896e9ad62ef72ef8f6ca670e755858880
--- /dev/null
+++ b/checkpoints/metadata_000004227072.json
@@ -0,0 +1 @@
+{"step": 129, "tokens_seen": 4227072, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 9.197491380414158}
\ No newline at end of file
diff --git a/checkpoints/metadata_000004653056.json b/checkpoints/metadata_000004653056.json
new file mode 100644
index 0000000000000000000000000000000000000000..166011578f7af9874860d8090f5428b5a1194c27
--- /dev/null
+++ b/checkpoints/metadata_000004653056.json
@@ -0,0 +1 @@
+{"step": 142, "tokens_seen": 4653056, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.989916327927832}
\ No newline at end of file
diff --git a/checkpoints/metadata_000005111808.json b/checkpoints/metadata_000005111808.json
new file mode 100644
index 0000000000000000000000000000000000000000..435930c83cba894878022b8d2bda15b37839db40
--- /dev/null
+++ b/checkpoints/metadata_000005111808.json
@@ -0,0 +1 @@
+{"step": 156, "tokens_seen": 5111808, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.789347695493596}
\ No newline at end of file
diff --git a/checkpoints/metadata_000005603328.json b/checkpoints/metadata_000005603328.json
new file mode 100644
index 0000000000000000000000000000000000000000..9cde0d1c0bbc7cb75619e207daf4845e05cb0f07
--- /dev/null
+++ b/checkpoints/metadata_000005603328.json
@@ -0,0 +1 @@
+{"step": 171, "tokens_seen": 5603328, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.60410839876213}
\ No newline at end of file
diff --git a/checkpoints/metadata_000006193152.json b/checkpoints/metadata_000006193152.json
new file mode 100644
index 0000000000000000000000000000000000000000..f41f2cc1b72fb8196a40f9ca9eb0e393fb40a9ae
--- /dev/null
+++ b/checkpoints/metadata_000006193152.json
@@ -0,0 +1 @@
+{"step": 189, "tokens_seen": 6193152, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.40378654180803}
\ No newline at end of file
diff --git a/checkpoints/metadata_000006782976.json b/checkpoints/metadata_000006782976.json
new file mode 100644
index 0000000000000000000000000000000000000000..7d1628b4f6590d0c445a2a45a7566eafc63a5f09
--- /dev/null
+++ b/checkpoints/metadata_000006782976.json
@@ -0,0 +1 @@
+{"step": 207, "tokens_seen": 6782976, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.23811376559379}
\ No newline at end of file
diff --git a/checkpoints/metadata_000007471104.json b/checkpoints/metadata_000007471104.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d5e5fbf5ed53eae1b3580129faf6ed1d730ef8c
--- /dev/null
+++ b/checkpoints/metadata_000007471104.json
@@ -0,0 +1 @@
+{"step": 228, "tokens_seen": 7471104, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 8.075189037614223}
\ No newline at end of file
diff --git a/checkpoints/metadata_000008224768.json b/checkpoints/metadata_000008224768.json
new file mode 100644
index 0000000000000000000000000000000000000000..553c91c9d68a403aec4908098b82abda9fbe1634
--- /dev/null
+++ b/checkpoints/metadata_000008224768.json
@@ -0,0 +1 @@
+{"step": 251, "tokens_seen": 8224768, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.918536753380052}
\ No newline at end of file
diff --git a/checkpoints/metadata_000009043968.json b/checkpoints/metadata_000009043968.json
new file mode 100644
index 0000000000000000000000000000000000000000..ddb226489a19216d2c1cdcdbc862a0804f800bb1
--- /dev/null
+++ b/checkpoints/metadata_000009043968.json
@@ -0,0 +1 @@
+{"step": 276, "tokens_seen": 9043968, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.788831272082824}
\ No newline at end of file
diff --git a/checkpoints/metadata_000009961472.json b/checkpoints/metadata_000009961472.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f91b832d428c7c6e555866e470cc0dd612a71f7
--- /dev/null
+++ b/checkpoints/metadata_000009961472.json
@@ -0,0 +1 @@
+{"step": 304, "tokens_seen": 9961472, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.66757438021451}
\ No newline at end of file
diff --git a/checkpoints/metadata_000010944512.json b/checkpoints/metadata_000010944512.json
new file mode 100644
index 0000000000000000000000000000000000000000..c874349355a59d72c8ee77add24bdf54ba910a99
--- /dev/null
+++ b/checkpoints/metadata_000010944512.json
@@ -0,0 +1 @@
+{"step": 334, "tokens_seen": 10944512, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.568145109932778}
\ No newline at end of file
diff --git a/checkpoints/metadata_000012058624.json b/checkpoints/metadata_000012058624.json
new file mode 100644
index 0000000000000000000000000000000000000000..567d7af5d0ea4368e99689e968246afd9dd932a3
--- /dev/null
+++ b/checkpoints/metadata_000012058624.json
@@ -0,0 +1 @@
+{"step": 368, "tokens_seen": 12058624, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.478137685161963}
\ No newline at end of file
diff --git a/checkpoints/metadata_000013271040.json b/checkpoints/metadata_000013271040.json
new file mode 100644
index 0000000000000000000000000000000000000000..124377810e458f9a955e3c609ccf8044cf7295f9
--- /dev/null
+++ b/checkpoints/metadata_000013271040.json
@@ -0,0 +1 @@
+{"step": 405, "tokens_seen": 13271040, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.3980350199178275}
\ No newline at end of file
diff --git a/checkpoints/metadata_000014581760.json b/checkpoints/metadata_000014581760.json
new file mode 100644
index 0000000000000000000000000000000000000000..a98ba3a6c16b141b5801dea26ae605730933859b
--- /dev/null
+++ b/checkpoints/metadata_000014581760.json
@@ -0,0 +1 @@
+{"step": 445, "tokens_seen": 14581760, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.3388065236413205}
\ No newline at end of file
diff --git a/checkpoints/metadata_000016056320.json b/checkpoints/metadata_000016056320.json
new file mode 100644
index 0000000000000000000000000000000000000000..31dac49bc1c5070d3ae803fa4a151b68b72d9e85
--- /dev/null
+++ b/checkpoints/metadata_000016056320.json
@@ -0,0 +1 @@
+{"step": 490, "tokens_seen": 16056320, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.2960175506969644}
\ No newline at end of file
diff --git a/checkpoints/metadata_000016384000.json b/checkpoints/metadata_000016384000.json
new file mode 100644
index 0000000000000000000000000000000000000000..0a04d0857f1d155c91d31d1554e5468ef378c0d7
--- /dev/null
+++ b/checkpoints/metadata_000016384000.json
@@ -0,0 +1 @@
+{"step": 500, "tokens_seen": 16384000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.2943454423626495}
\ No newline at end of file
diff --git a/checkpoints/metadata_000017661952.json b/checkpoints/metadata_000017661952.json
new file mode 100644
index 0000000000000000000000000000000000000000..ac41547bf57c7a898b68cfdfdd948d4f40b1f3d9
--- /dev/null
+++ b/checkpoints/metadata_000017661952.json
@@ -0,0 +1 @@
+{"step": 539, "tokens_seen": 17661952, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.267293551259387}
\ No newline at end of file
diff --git a/checkpoints/metadata_000019431424.json b/checkpoints/metadata_000019431424.json
new file mode 100644
index 0000000000000000000000000000000000000000..7c916aca728364bdf41f924947423081b7ce9dc7
--- /dev/null
+++ b/checkpoints/metadata_000019431424.json
@@ -0,0 +1 @@
+{"step": 593, "tokens_seen": 19431424, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.234690626814807}
\ No newline at end of file
diff --git a/checkpoints/metadata_000021364736.json b/checkpoints/metadata_000021364736.json
new file mode 100644
index 0000000000000000000000000000000000000000..928e62955fc001378afaad4242b57f86a4e781e8
--- /dev/null
+++ b/checkpoints/metadata_000021364736.json
@@ -0,0 +1 @@
+{"step": 652, "tokens_seen": 21364736, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.203540739078531}
\ No newline at end of file
diff --git a/checkpoints/metadata_000023494656.json b/checkpoints/metadata_000023494656.json
new file mode 100644
index 0000000000000000000000000000000000000000..9eb22c507e91f823a9647a5bc32bcc6776381383
--- /dev/null
+++ b/checkpoints/metadata_000023494656.json
@@ -0,0 +1 @@
+{"step": 717, "tokens_seen": 23494656, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.184632052646438}
\ No newline at end of file
diff --git a/checkpoints/metadata_000025853952.json b/checkpoints/metadata_000025853952.json
new file mode 100644
index 0000000000000000000000000000000000000000..208280eaa46c5570d6f0b5bdbb70323d2a58147f
--- /dev/null
+++ b/checkpoints/metadata_000025853952.json
@@ -0,0 +1 @@
+{"step": 789, "tokens_seen": 25853952, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.161264433218609}
\ No newline at end of file
diff --git a/checkpoints/metadata_000028442624.json b/checkpoints/metadata_000028442624.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd89e4147c263f71d60720be1aac2bcaa2a1e0e9
--- /dev/null
+++ b/checkpoints/metadata_000028442624.json
@@ -0,0 +1 @@
+{"step": 868, "tokens_seen": 28442624, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.141852931468452}
\ No newline at end of file
diff --git a/checkpoints/metadata_000031293440.json b/checkpoints/metadata_000031293440.json
new file mode 100644
index 0000000000000000000000000000000000000000..e25844ee56e4145b1a59d103022b8d83d6ec2f49
--- /dev/null
+++ b/checkpoints/metadata_000031293440.json
@@ -0,0 +1 @@
+{"step": 955, "tokens_seen": 31293440, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.130734166348351}
\ No newline at end of file
diff --git a/checkpoints/metadata_000032768000.json b/checkpoints/metadata_000032768000.json
new file mode 100644
index 0000000000000000000000000000000000000000..5c7945defb75171081f815b2fd7a4bf5630e3b14
--- /dev/null
+++ b/checkpoints/metadata_000032768000.json
@@ -0,0 +1 @@
+{"step": 1000, "tokens_seen": 32768000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.114958265617314}
\ No newline at end of file
diff --git a/checkpoints/metadata_000034439168.json b/checkpoints/metadata_000034439168.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d59c747ce2ed6cbec9a47a0c5a3eed7b6272717
--- /dev/null
+++ b/checkpoints/metadata_000034439168.json
@@ -0,0 +1 @@
+{"step": 1051, "tokens_seen": 34439168, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.115081387842345}
\ No newline at end of file
diff --git a/checkpoints/metadata_000037879808.json b/checkpoints/metadata_000037879808.json
new file mode 100644
index 0000000000000000000000000000000000000000..90f770c2988385d0d4e5a06bc6951667e760968a
--- /dev/null
+++ b/checkpoints/metadata_000037879808.json
@@ -0,0 +1 @@
+{"step": 1156, "tokens_seen": 37879808, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.0861636030099024}
\ No newline at end of file
diff --git a/checkpoints/metadata_000041648128.json b/checkpoints/metadata_000041648128.json
new file mode 100644
index 0000000000000000000000000000000000000000..ad6cc89be15cb7ac8012f35389a57235ab0d0827
--- /dev/null
+++ b/checkpoints/metadata_000041648128.json
@@ -0,0 +1 @@
+{"step": 1271, "tokens_seen": 41648128, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.073608590465179}
\ No newline at end of file
diff --git a/checkpoints/metadata_000045842432.json b/checkpoints/metadata_000045842432.json
new file mode 100644
index 0000000000000000000000000000000000000000..e30edf3804097ce02862f2b21a65f66dd172cb13
--- /dev/null
+++ b/checkpoints/metadata_000045842432.json
@@ -0,0 +1 @@
+{"step": 1399, "tokens_seen": 45842432, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.046477499088217}
\ No newline at end of file
diff --git a/checkpoints/metadata_000049152000.json b/checkpoints/metadata_000049152000.json
new file mode 100644
index 0000000000000000000000000000000000000000..8ffd2fd32d909480e6477a9719a45087baf083c0
--- /dev/null
+++ b/checkpoints/metadata_000049152000.json
@@ -0,0 +1 @@
+{"step": 1500, "tokens_seen": 49152000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.025890791618269}
\ No newline at end of file
diff --git a/checkpoints/metadata_000050397184.json b/checkpoints/metadata_000050397184.json
new file mode 100644
index 0000000000000000000000000000000000000000..2bce6ca43c3732f76b50fa607e7a45421c7e1bd9
--- /dev/null
+++ b/checkpoints/metadata_000050397184.json
@@ -0,0 +1 @@
+{"step": 1538, "tokens_seen": 50397184, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.021497966023416}
\ No newline at end of file
diff --git a/checkpoints/metadata_000055443456.json b/checkpoints/metadata_000055443456.json
new file mode 100644
index 0000000000000000000000000000000000000000..08cdc878a98d3dbec451638fd364c2a3ce483b8a
--- /dev/null
+++ b/checkpoints/metadata_000055443456.json
@@ -0,0 +1 @@
+{"step": 1692, "tokens_seen": 55443456, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 7.011519785801487}
\ No newline at end of file
diff --git a/checkpoints/metadata_000061014016.json b/checkpoints/metadata_000061014016.json
new file mode 100644
index 0000000000000000000000000000000000000000..5223c8a447ebf6bc2fee6b0aabb9146439eb5885
--- /dev/null
+++ b/checkpoints/metadata_000061014016.json
@@ -0,0 +1 @@
+{"step": 1862, "tokens_seen": 61014016, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.993219627097563}
\ No newline at end of file
diff --git a/checkpoints/metadata_000065536000.json b/checkpoints/metadata_000065536000.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd166fc7a10706c606078980d3729e2fb52f2d3c
--- /dev/null
+++ b/checkpoints/metadata_000065536000.json
@@ -0,0 +1 @@
+{"step": 2000, "tokens_seen": 65536000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.971705233151438}
\ No newline at end of file
diff --git a/checkpoints/metadata_000067108864.json b/checkpoints/metadata_000067108864.json
new file mode 100644
index 0000000000000000000000000000000000000000..e4f5e08665a2ccbc2bede22560a4568459fe8bcc
--- /dev/null
+++ b/checkpoints/metadata_000067108864.json
@@ -0,0 +1 @@
+{"step": 2048, "tokens_seen": 67108864, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.957607114329103}
\ No newline at end of file
diff --git a/checkpoints/metadata_000073826304.json b/checkpoints/metadata_000073826304.json
new file mode 100644
index 0000000000000000000000000000000000000000..2950d1f879aff21d0407d3810674a434f5c8ebe4
--- /dev/null
+++ b/checkpoints/metadata_000073826304.json
@@ -0,0 +1 @@
+{"step": 2253, "tokens_seen": 73826304, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.937125526864601}
\ No newline at end of file
diff --git a/checkpoints/metadata_000081199104.json b/checkpoints/metadata_000081199104.json
new file mode 100644
index 0000000000000000000000000000000000000000..c055cde3f9eb3196a78c56170d1c5da0e595f8fd
--- /dev/null
+++ b/checkpoints/metadata_000081199104.json
@@ -0,0 +1 @@
+{"step": 2478, "tokens_seen": 81199104, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.919170392615817}
\ No newline at end of file
diff --git a/checkpoints/metadata_000081920000.json b/checkpoints/metadata_000081920000.json
new file mode 100644
index 0000000000000000000000000000000000000000..5bd4bf1eed47d9e04be6841b38302c10e7e2e21f
--- /dev/null
+++ b/checkpoints/metadata_000081920000.json
@@ -0,0 +1 @@
+{"step": 2500, "tokens_seen": 81920000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.918353411055644}
\ No newline at end of file
diff --git a/checkpoints/metadata_000089325568.json b/checkpoints/metadata_000089325568.json
new file mode 100644
index 0000000000000000000000000000000000000000..481ec574c2680fc814588904e21c5228c3d08f04
--- /dev/null
+++ b/checkpoints/metadata_000089325568.json
@@ -0,0 +1 @@
+{"step": 2726, "tokens_seen": 89325568, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.8857921812298475}
\ No newline at end of file
diff --git a/checkpoints/metadata_000098271232.json b/checkpoints/metadata_000098271232.json
new file mode 100644
index 0000000000000000000000000000000000000000..b8b6d291ebc49a2a99048bcfa7c9ce2f678c7f59
--- /dev/null
+++ b/checkpoints/metadata_000098271232.json
@@ -0,0 +1 @@
+{"step": 2999, "tokens_seen": 98271232, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.836342736631777}
\ No newline at end of file
diff --git a/checkpoints/metadata_000098304000.json b/checkpoints/metadata_000098304000.json
new file mode 100644
index 0000000000000000000000000000000000000000..f3b85e31338af58632729921f9c9c1c6e678b503
--- /dev/null
+++ b/checkpoints/metadata_000098304000.json
@@ -0,0 +1 @@
+{"step": 3000, "tokens_seen": 98304000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.8361660596774465}
\ No newline at end of file
diff --git a/checkpoints/metadata_000108068864.json b/checkpoints/metadata_000108068864.json
new file mode 100644
index 0000000000000000000000000000000000000000..49e5d5dcf63063b28f4172fda1bc073a940896ce
--- /dev/null
+++ b/checkpoints/metadata_000108068864.json
@@ -0,0 +1 @@
+{"step": 3298, "tokens_seen": 108068864, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.622358717633461}
\ No newline at end of file
diff --git a/checkpoints/metadata_000114688000.json b/checkpoints/metadata_000114688000.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7a5298587665deb080b17a1f299ee2646baca8c
--- /dev/null
+++ b/checkpoints/metadata_000114688000.json
@@ -0,0 +1 @@
+{"step": 3500, "tokens_seen": 114688000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.4845515582650854}
\ No newline at end of file
diff --git a/checkpoints/metadata_000118882304.json b/checkpoints/metadata_000118882304.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0af7089dd254a91707cbbc3b82627f760e310b
--- /dev/null
+++ b/checkpoints/metadata_000118882304.json
@@ -0,0 +1 @@
+{"step": 3628, "tokens_seen": 118882304, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.415539012938077}
\ No newline at end of file
diff --git a/checkpoints/metadata_000130777088.json b/checkpoints/metadata_000130777088.json
new file mode 100644
index 0000000000000000000000000000000000000000..e342b1979863d21e59ae0afad02ad2bf7fc83ee3
--- /dev/null
+++ b/checkpoints/metadata_000130777088.json
@@ -0,0 +1 @@
+{"step": 3991, "tokens_seen": 130777088, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.361387064652267}
\ No newline at end of file
diff --git a/checkpoints/metadata_000131072000.json b/checkpoints/metadata_000131072000.json
new file mode 100644
index 0000000000000000000000000000000000000000..561b7815ba390176b53f62d7fb208246fd2df8ea
--- /dev/null
+++ b/checkpoints/metadata_000131072000.json
@@ -0,0 +1 @@
+{"step": 4000, "tokens_seen": 131072000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.353944947282875}
\ No newline at end of file
diff --git a/checkpoints/metadata_000143851520.json b/checkpoints/metadata_000143851520.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e334512e728c8a4040348f9c0c78ef911f410fb
--- /dev/null
+++ b/checkpoints/metadata_000143851520.json
@@ -0,0 +1 @@
+{"step": 4390, "tokens_seen": 143851520, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.278724554869647}
\ No newline at end of file
diff --git a/checkpoints/metadata_000147456000.json b/checkpoints/metadata_000147456000.json
new file mode 100644
index 0000000000000000000000000000000000000000..a86ae85c0c91d9eaaa7a2a48e4ffabb59e093264
--- /dev/null
+++ b/checkpoints/metadata_000147456000.json
@@ -0,0 +1 @@
+{"step": 4500, "tokens_seen": 147456000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.254443075349619}
\ No newline at end of file
diff --git a/checkpoints/metadata_000158269440.json b/checkpoints/metadata_000158269440.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5d44dbcb3cdece5d3169e8cf485cfd7137efbe4
--- /dev/null
+++ b/checkpoints/metadata_000158269440.json
@@ -0,0 +1 @@
+{"step": 4830, "tokens_seen": 158269440, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.244561346431229}
\ No newline at end of file
diff --git a/checkpoints/metadata_000163840000.json b/checkpoints/metadata_000163840000.json
new file mode 100644
index 0000000000000000000000000000000000000000..87aace25d8aee7f4ac2956d7d403600e62553815
--- /dev/null
+++ b/checkpoints/metadata_000163840000.json
@@ -0,0 +1 @@
+{"step": 5000, "tokens_seen": 163840000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.2684190183297055}
\ No newline at end of file
diff --git a/checkpoints/metadata_000174096384.json b/checkpoints/metadata_000174096384.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9afaef0eea3b5ace828e656dc2980fc8966d8db
--- /dev/null
+++ b/checkpoints/metadata_000174096384.json
@@ -0,0 +1 @@
+{"step": 5313, "tokens_seen": 174096384, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.219244561676171}
\ No newline at end of file
diff --git a/checkpoints/metadata_000180224000.json b/checkpoints/metadata_000180224000.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6abd98a921f18c65d62ba4e402f62ca14a0e5b5
--- /dev/null
+++ b/checkpoints/metadata_000180224000.json
@@ -0,0 +1 @@
+{"step": 5500, "tokens_seen": 180224000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.245759664872331}
\ No newline at end of file
diff --git a/checkpoints/metadata_000191496192.json b/checkpoints/metadata_000191496192.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ea8ab125089741fbd2b70315865085ec4c426f6
--- /dev/null
+++ b/checkpoints/metadata_000191496192.json
@@ -0,0 +1 @@
+{"step": 5844, "tokens_seen": 191496192, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.208193731745974}
\ No newline at end of file
diff --git a/checkpoints/metadata_000196608000.json b/checkpoints/metadata_000196608000.json
new file mode 100644
index 0000000000000000000000000000000000000000..b94df576f4c1982f26b4aa7736cd4a95b190c3f2
--- /dev/null
+++ b/checkpoints/metadata_000196608000.json
@@ -0,0 +1 @@
+{"step": 6000, "tokens_seen": 196608000, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.202362959679445}
\ No newline at end of file
diff --git a/checkpoints/metadata_000196706304.json b/checkpoints/metadata_000196706304.json
new file mode 100644
index 0000000000000000000000000000000000000000..a14eca08695925cb9542100542e4ffad8d1a8f5b
--- /dev/null
+++ b/checkpoints/metadata_000196706304.json
@@ -0,0 +1 @@
+{"step": 6003, "tokens_seen": 196706304, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.201176688586682}
\ No newline at end of file
diff --git a/checkpoints/metadata_000197361664.json b/checkpoints/metadata_000197361664.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f905d31b5bd17c57ed7065ecc8399bd3d82d377
--- /dev/null
+++ b/checkpoints/metadata_000197361664.json
@@ -0,0 +1 @@
+{"step": 6023, "tokens_seen": 197361664, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.190655033893243}
\ No newline at end of file
diff --git a/checkpoints/metadata_000198017024.json b/checkpoints/metadata_000198017024.json
new file mode 100644
index 0000000000000000000000000000000000000000..46889143464f809b6f2b1b8b23d81a55332a85bd
--- /dev/null
+++ b/checkpoints/metadata_000198017024.json
@@ -0,0 +1 @@
+{"step": 6043, "tokens_seen": 198017024, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.19215985466722}
\ No newline at end of file
diff --git a/checkpoints/metadata_000198672384.json b/checkpoints/metadata_000198672384.json
new file mode 100644
index 0000000000000000000000000000000000000000..fc5446d9e45d32db9066342ab3cae65f06c3047e
--- /dev/null
+++ b/checkpoints/metadata_000198672384.json
@@ -0,0 +1 @@
+{"step": 6063, "tokens_seen": 198672384, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.184825366183369}
\ No newline at end of file
diff --git a/checkpoints/metadata_000199327744.json b/checkpoints/metadata_000199327744.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7087815f6d295da4276866dea313ab86c91b7cb
--- /dev/null
+++ b/checkpoints/metadata_000199327744.json
@@ -0,0 +1 @@
+{"step": 6083, "tokens_seen": 199327744, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.188555257502236}
\ No newline at end of file
diff --git a/checkpoints/metadata_000199950336.json b/checkpoints/metadata_000199950336.json
new file mode 100644
index 0000000000000000000000000000000000000000..6650c37319f0a1f0bd58ee3d44267d41f87216a4
--- /dev/null
+++ b/checkpoints/metadata_000199950336.json
@@ -0,0 +1 @@
+{"step": 6102, "tokens_seen": 199950336, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.176267960488811}
\ No newline at end of file
diff --git a/checkpoints/model_weights_000000032768.pt b/checkpoints/model_weights_000000032768.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03511d91e914d121706074672ed3717e4b31658a
--- /dev/null
+++ b/checkpoints/model_weights_000000032768.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78ce67efbdfc3c9279831326e38cbb39268eb87b8627e4d8e3239dbba09c7ac9
+size 225208789
diff --git a/checkpoints/model_weights_000000327680.pt b/checkpoints/model_weights_000000327680.pt
new file mode 100644
index 0000000000000000000000000000000000000000..391feaf49e26a97180debc37029bb8db5acf902a
--- /dev/null
+++ b/checkpoints/model_weights_000000327680.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c553516cb4efb78a5d59574af85fac342a806a7636995a28f427c49241d02551
+size 225208789
diff --git a/checkpoints/model_weights_000000360448.pt b/checkpoints/model_weights_000000360448.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f51101f94059ae4bbb799316ebb7221251016f41
--- /dev/null
+++ b/checkpoints/model_weights_000000360448.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb45cfd6139dc612b92fd0fe4192481c9ba50cfa78017d6750c540b376a0a97
+size 225208789
diff --git a/checkpoints/model_weights_000000425984.pt b/checkpoints/model_weights_000000425984.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d09b41d2a96881bfb593f7c1651a33217607c023
--- /dev/null
+++ b/checkpoints/model_weights_000000425984.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53eb3c50adcdcd7f4ac5ce5b0c4dd8e76d63bd3613d8ca9e0ca54714c285ee59
+size 225208789
diff --git a/checkpoints/model_weights_000000458752.pt b/checkpoints/model_weights_000000458752.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a6477f586e185934dd69344da89b033ebf476535
--- /dev/null
+++ b/checkpoints/model_weights_000000458752.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17522c3ecfda4580b31d00f3987d1f9fcb70ad479a9673c85e5149b7d162155b
+size 225208789
diff --git a/checkpoints/model_weights_000000491520.pt b/checkpoints/model_weights_000000491520.pt
new file mode 100644
index 0000000000000000000000000000000000000000..851c701ec09a4648ca7fb14de645956cfe6dd7a8
--- /dev/null
+++ b/checkpoints/model_weights_000000491520.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2fb33378c3a6d40711e4f56d8b480f0fb6b5a344b48947ee40b10165dd49f4c
+size 225208789
diff --git a/checkpoints/model_weights_000000557056.pt b/checkpoints/model_weights_000000557056.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a68770feec5568fbe8ffb2e70800d5e657ec68f
--- /dev/null
+++ b/checkpoints/model_weights_000000557056.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77696d0904eaa9a80d28799a0a82f515dc9b8c81a1b61ca18548c55a378e2b8d
+size 225208789
diff --git a/checkpoints/model_weights_000000622592.pt b/checkpoints/model_weights_000000622592.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eab5f20067bc21b0bf7d3ff83d54fab7bc8edf36
--- /dev/null
+++ b/checkpoints/model_weights_000000622592.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a07d03617aaf5a970a57de129f136e517a9b016c8e77814116c50f9943e2bee4
+size 225208789
diff --git a/checkpoints/model_weights_000000688128.pt b/checkpoints/model_weights_000000688128.pt
new file mode 100644
index 0000000000000000000000000000000000000000..254f3f0d87f8c455564ea1ba9e551b26df44e4fb
--- /dev/null
+++ b/checkpoints/model_weights_000000688128.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db76c910298c4b280f19bbdea22ac60b97657283942e03ef41e474401e28f293
+size 225208789
diff --git a/checkpoints/model_weights_000000753664.pt b/checkpoints/model_weights_000000753664.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3313abc0522786dae061b82c3be3c4d0b8d8bdd6
--- /dev/null
+++ b/checkpoints/model_weights_000000753664.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80177ffac533939478b0478f9bb84fe503565fdb77ebfe21efb828d24d338d20
+size 225208789
diff --git a/checkpoints/model_weights_000000819200.pt b/checkpoints/model_weights_000000819200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fe24816c21861276b457b3a77cabce0ecea4ece
--- /dev/null
+++ b/checkpoints/model_weights_000000819200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:909ce0f6a4f97c66001b8269c6989b5ff990fe4de9d5c7d634982603d5ba48b9
+size 225208789
diff --git a/checkpoints/model_weights_000000917504.pt b/checkpoints/model_weights_000000917504.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3ae86dd4441a5be520aa9416c8d0673d0082ec2
--- /dev/null
+++ b/checkpoints/model_weights_000000917504.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91d626bd914cd7a3382487fa972c4b00dd82ba15b426fe0db2377450b053b432
+size 225208789
diff --git a/checkpoints/model_weights_000000983040.pt b/checkpoints/model_weights_000000983040.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f462d2c72862bf2e5ec6333d79ae270235f98d1e
--- /dev/null
+++ b/checkpoints/model_weights_000000983040.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aa3c501b7f2fa375eed55dbedf3ebc90ef74809a94d8089d82414c1d414ec54
+size 225208789
diff --git a/checkpoints/model_weights_000001114112.pt b/checkpoints/model_weights_000001114112.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b33d4f074f6618088ed855f515d5e5ea0cad911c
--- /dev/null
+++ b/checkpoints/model_weights_000001114112.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c05e873527cac9f28f2f08ca2de33d5778f7bb138cf42d435973705328a11709
+size 225208789
diff --git a/checkpoints/model_weights_000001212416.pt b/checkpoints/model_weights_000001212416.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cab8da2c400c871647fb0140abc8f152836028c1
--- /dev/null
+++ b/checkpoints/model_weights_000001212416.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c15bce4e74d699bda9755acc7fd6d1db6528e641ea57338e23207f4af764a1d3
+size 225208789
diff --git a/checkpoints/model_weights_000001343488.pt b/checkpoints/model_weights_000001343488.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42df2960c40f8b3ea611cac2a62ecd5efdcf3044
--- /dev/null
+++ b/checkpoints/model_weights_000001343488.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c29f998d6af5b45c668d556f2eb97f3c0c023e60a226aadd9740bbe9bc6a474e
+size 225208789
diff --git a/checkpoints/model_weights_000001474560.pt b/checkpoints/model_weights_000001474560.pt
new file mode 100644
index 0000000000000000000000000000000000000000..867166ff3c82912933434d64c0dc72e6144024da
--- /dev/null
+++ b/checkpoints/model_weights_000001474560.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6283678b9d80a59172bcd442a1834f774521e52d64c63ad1aa41a2f5b627665
+size 225208789
diff --git a/checkpoints/model_weights_000001605632.pt b/checkpoints/model_weights_000001605632.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9574dbf5e5f6aa0fb155a1200a561ec178cf6d4
--- /dev/null
+++ b/checkpoints/model_weights_000001605632.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677076ed157344aa42722369831f5af2f7ebe0684e9bd472bc0515c33d667869
+size 225208789
diff --git a/checkpoints/model_weights_000001769472.pt b/checkpoints/model_weights_000001769472.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cb4c0dba5bd58a31ea71783dc6a1f91bf3d916d
--- /dev/null
+++ b/checkpoints/model_weights_000001769472.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d7bf2cab28842983d32af65a66ba797964994bc98bba837bc5c1f094ed8f6e6
+size 225208789
diff --git a/checkpoints/model_weights_000001966080.pt b/checkpoints/model_weights_000001966080.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9fec7e0d7710845dbb19e42ccb30bea2464ad7f5
--- /dev/null
+++ b/checkpoints/model_weights_000001966080.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87dfff73380d3a17fef56dce1a822d563637d703c50bd23b03d7eb20c97f4abf
+size 225208789
diff --git a/checkpoints/model_weights_000002162688.pt b/checkpoints/model_weights_000002162688.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a4ddabf2fbfff294c1e4dc6efd1dea602dc2e99
--- /dev/null
+++ b/checkpoints/model_weights_000002162688.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6336146703b878b4d0180b73a4702f187f93e0adcf16ad956ac22c962d9f3dc6
+size 225208789
diff --git a/checkpoints/model_weights_000002359296.pt b/checkpoints/model_weights_000002359296.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9b4fce9b422e3c61dda98cd8ca531d1af26b71e
--- /dev/null
+++ b/checkpoints/model_weights_000002359296.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76136f118fc4550434c5c442872444a5d8abb91200e4a428f4484c75a01b9c3f
+size 225208789
diff --git a/checkpoints/model_weights_000002621440.pt b/checkpoints/model_weights_000002621440.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9a86ed53f00a08b600bf4434e89f16b2bcbc87bc
--- /dev/null
+++ b/checkpoints/model_weights_000002621440.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5402f002252d6ccec1a2b6383808125a13cc165e7682dfbf789c1aa51754118e
+size 225208789
diff --git a/checkpoints/model_weights_000002883584.pt b/checkpoints/model_weights_000002883584.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70ed6083f84e21eb75ae0bcd0be48a88f72bd084
--- /dev/null
+++ b/checkpoints/model_weights_000002883584.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edd5c5b0c3e2ed3da06117be5ef81a80c43cde322537651277e19bbc495bb1bf
+size 225208789
diff --git a/checkpoints/model_weights_000003178496.pt b/checkpoints/model_weights_000003178496.pt
new file mode 100644
index 0000000000000000000000000000000000000000..482685c3ca7c6ffc78129d0b40b66022ff8e9aca
--- /dev/null
+++ b/checkpoints/model_weights_000003178496.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de6d8897fdffbd9451b7cdb7bb4c2676727523ce36499fa71ac61b18ca1670f7
+size 225208789
diff --git a/checkpoints/model_weights_000003473408.pt b/checkpoints/model_weights_000003473408.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe377b371976245ee7fbf0afb3b767784fbdcda8
--- /dev/null
+++ b/checkpoints/model_weights_000003473408.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:270e77e3f3540e48d19489f33e7f32f52205ed5c415bb754184731cad0e71bd2
+size 225208789
diff --git a/checkpoints/model_weights_000003833856.pt b/checkpoints/model_weights_000003833856.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d21f2228dedc752178fc03263d30f4aebead8d01
--- /dev/null
+++ b/checkpoints/model_weights_000003833856.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b384ecc8adb56d75f2740fe757ebc36f8a62b5d8d16ff2f77e05f5e2b83b4396
+size 225208789
diff --git a/checkpoints/model_weights_000004227072.pt b/checkpoints/model_weights_000004227072.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02a956683971247092cc33855f7c967b52cba13f
--- /dev/null
+++ b/checkpoints/model_weights_000004227072.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b89339a4bf510d44c25be810c37b715ede3f4474f95283f77d61228f6ed2706a
+size 225208789
diff --git a/checkpoints/model_weights_000004653056.pt b/checkpoints/model_weights_000004653056.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76a05613fbb68e2b380364e7ec7b986f1849435c
--- /dev/null
+++ b/checkpoints/model_weights_000004653056.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d24f19def14364f050cf98fe6cf688d99ef9a0e08ab07e7a4782aa2698e066b6
+size 225208789
diff --git a/checkpoints/model_weights_000005111808.pt b/checkpoints/model_weights_000005111808.pt
new file mode 100644
index 0000000000000000000000000000000000000000..52a4f4787937fe5699edf7b969b166b417a9f71a
--- /dev/null
+++ b/checkpoints/model_weights_000005111808.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7069dc8f4b970cd07ffe09f6f19d0ac35d867f1e79360fd5cedfafa8cd673d6
+size 225208789
diff --git a/checkpoints/model_weights_000005603328.pt b/checkpoints/model_weights_000005603328.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c3b98b02f356479d88bdeef6c946916993d3246
--- /dev/null
+++ b/checkpoints/model_weights_000005603328.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4c22e9815b6433bba4f4292af7540549b948490a8a274ec95f0abf303492afe
+size 225208789
diff --git a/checkpoints/model_weights_000006193152.pt b/checkpoints/model_weights_000006193152.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1adce99a30f09f1d01f55f3e440753b8ff21607d
--- /dev/null
+++ b/checkpoints/model_weights_000006193152.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0e9e1e7d4d9f71cc6c4a8ca64ea5a984e13ffd0877a315b71500c414134c168
+size 225208789
diff --git a/checkpoints/model_weights_000006782976.pt b/checkpoints/model_weights_000006782976.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c750f6b39b3498306b1510a97863ba702a797338
--- /dev/null
+++ b/checkpoints/model_weights_000006782976.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:334886feb95a40c9108eafd796c8c26b13f5fe33406303dae7306cd96335b32e
+size 225208789
diff --git a/checkpoints/model_weights_000007471104.pt b/checkpoints/model_weights_000007471104.pt
new file mode 100644
index 0000000000000000000000000000000000000000..21a929bfcb2332c9d4fc52fc520cc4708bf1c03f
--- /dev/null
+++ b/checkpoints/model_weights_000007471104.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d35c05d6fc524daf8b6c9341d4e25b2b3fd232f03c2312c24c58a1524ed4bc
+size 225208789
diff --git a/checkpoints/model_weights_000008224768.pt b/checkpoints/model_weights_000008224768.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9755c9146e9460686c43a2e7e2448070e8dc098
--- /dev/null
+++ b/checkpoints/model_weights_000008224768.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb845f1ce25a112874ae640633ae027a269258f17836c2b450106fcbd7a743fc
+size 225208789
diff --git a/checkpoints/model_weights_000009043968.pt b/checkpoints/model_weights_000009043968.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1cd5451a290c0b2bbf41774d2565d60cc6e09b6d
--- /dev/null
+++ b/checkpoints/model_weights_000009043968.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18aa500aad49b4254207a140e74134cea498dc41d3d6788327c9a6eec9897c7e
+size 225208789
diff --git a/checkpoints/model_weights_000009961472.pt b/checkpoints/model_weights_000009961472.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64eb0d2ac6eaf688730f8a139c1865c93aa54e84
--- /dev/null
+++ b/checkpoints/model_weights_000009961472.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73c3d603c933d5f40cd951e242f817064f5d73f15f0f57a309ff182f32a5540f
+size 225208789
diff --git a/checkpoints/model_weights_000010944512.pt b/checkpoints/model_weights_000010944512.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d19de292c7a4d088666ff6a6b58b2bc48bc8610
--- /dev/null
+++ b/checkpoints/model_weights_000010944512.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2de0119ae5a033522fd2904b3d42fcdd68347d66e66061d3c6e4ddb4baa489f
+size 225208789
diff --git a/checkpoints/model_weights_000012058624.pt b/checkpoints/model_weights_000012058624.pt
new file mode 100644
index 0000000000000000000000000000000000000000..53fb3eeff8f0cfd9ce057ac857c763a6b6c1b90c
--- /dev/null
+++ b/checkpoints/model_weights_000012058624.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee950b9c95cd1a7395e865d62632bb6ef89fbef743d2e81fbe2eb0d20b7bcc5
+size 225208789
diff --git a/checkpoints/model_weights_000013271040.pt b/checkpoints/model_weights_000013271040.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1feebb6de62b41a408b4bd097399cd797410d390
--- /dev/null
+++ b/checkpoints/model_weights_000013271040.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c222d52f0762615b46dc0ccbde971c35b64a31cef6a1450e1a947a5d5b11b1a
+size 225208789
diff --git a/checkpoints/model_weights_000014581760.pt b/checkpoints/model_weights_000014581760.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5b730f3531799b24676d5036283954b8e4222c1
--- /dev/null
+++ b/checkpoints/model_weights_000014581760.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:565dd37cc0a7b930f070a16e571da463bf65e84875aaa4a446a13d66314b2528
+size 225208789
diff --git a/checkpoints/model_weights_000016056320.pt b/checkpoints/model_weights_000016056320.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70efb82baf1f97fd8e83a02e20669231c6258dc5
--- /dev/null
+++ b/checkpoints/model_weights_000016056320.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7da82cf98ad139530d2af2d84166bea443f3aeedcac75d18b3b96a1f5a201b9a
+size 225208789
diff --git a/checkpoints/model_weights_000016384000.pt b/checkpoints/model_weights_000016384000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..979acf7f068c85afed5d941b9b7ae79939847bc4
--- /dev/null
+++ b/checkpoints/model_weights_000016384000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbac8751fe02856b6ab67bc6ec47643f0193902029daab589a4c2b379446029f
+size 225208789
diff --git a/checkpoints/model_weights_000017661952.pt b/checkpoints/model_weights_000017661952.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99ca61fbb9e030b5197cae7d55e5e811db2c81f9
--- /dev/null
+++ b/checkpoints/model_weights_000017661952.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a26b0d3ba016d6443c09c54541aa53f24ff5899885848fbfd7640385461de09
+size 225208789
diff --git a/checkpoints/model_weights_000019431424.pt b/checkpoints/model_weights_000019431424.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd37d6601b235228c1b19900feddc02dc9dff3ec
--- /dev/null
+++ b/checkpoints/model_weights_000019431424.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:233fbdf4a030039fbc2595d54c3ced020210f5f34d9640f2d67c426d9c162bd5
+size 225208789
diff --git a/checkpoints/model_weights_000021364736.pt b/checkpoints/model_weights_000021364736.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74a497312b7fa003d3fed14cb3833acf6b5f9f44
--- /dev/null
+++ b/checkpoints/model_weights_000021364736.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21a4b677b1b9b1c466d294c7398328510c610d257a25f5ce586bd72824141d9f
+size 225208789
diff --git a/checkpoints/model_weights_000023494656.pt b/checkpoints/model_weights_000023494656.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ca28f3698e79d639c4d0d03f4efcb0c06465450a
--- /dev/null
+++ b/checkpoints/model_weights_000023494656.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6b5aa272e673992be40ecb1eef8b526e942d9d872e69ed93c214a8ff3d87d8d
+size 225208789
diff --git a/checkpoints/model_weights_000025853952.pt b/checkpoints/model_weights_000025853952.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26f66ae61bd04e37cd97a021ca501b449d8a6661
--- /dev/null
+++ b/checkpoints/model_weights_000025853952.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebeab5881d945d994733b751c5ae50256f34550a29248438ad9400b5cf09cf8d
+size 225208789
diff --git a/checkpoints/model_weights_000028442624.pt b/checkpoints/model_weights_000028442624.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b42eeff60a823264c43a2cfa79580cb3835eacdf
--- /dev/null
+++ b/checkpoints/model_weights_000028442624.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39fd11aa8059fc86d663d720b23463310902d62c2d4a93ebea3b9d23522a6fbe
+size 225208789
diff --git a/checkpoints/model_weights_000031293440.pt b/checkpoints/model_weights_000031293440.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3bf2784a4675bb1393506b1b9a9177638b94fba5
--- /dev/null
+++ b/checkpoints/model_weights_000031293440.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2b51697f6e14d2949985957eda7161f1f3b955b68b879271bff58e29e048bfe
+size 225208789
diff --git a/checkpoints/model_weights_000032768000.pt b/checkpoints/model_weights_000032768000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fce8c43d40c87c7ff280a8d824cc36ba042d82e
--- /dev/null
+++ b/checkpoints/model_weights_000032768000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd63bd9833073935660d546c0287eb425621aec375bb95b4182522ddba602ebb
+size 225208789
diff --git a/checkpoints/model_weights_000034439168.pt b/checkpoints/model_weights_000034439168.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2f9173fcf2da44ac862cf3b8dcfeab6c2c37878
--- /dev/null
+++ b/checkpoints/model_weights_000034439168.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f600a2c02398b4146a4fa267db8710393057ae3b2511de6ec4fcb5d28b75b8cd
+size 225208789
diff --git a/checkpoints/model_weights_000037879808.pt b/checkpoints/model_weights_000037879808.pt
new file mode 100644
index 0000000000000000000000000000000000000000..832a65a43d3360cc702026b00667b807644adc6c
--- /dev/null
+++ b/checkpoints/model_weights_000037879808.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bc27350fc9574397ad3cbc48b8691c51c7dc3e98042cdceb143337fd5012cbd
+size 225208789
diff --git a/checkpoints/model_weights_000041648128.pt b/checkpoints/model_weights_000041648128.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4f441303532e1d6e94c862052744b092dd1141b2
--- /dev/null
+++ b/checkpoints/model_weights_000041648128.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8637742273aaab9c0c05d20d8f84cc78bd4ee6c63067d04c0319b051031a33
+size 225208789
diff --git a/checkpoints/model_weights_000045842432.pt b/checkpoints/model_weights_000045842432.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb7c33e6f099f0261f6f60ec6aea79991d0af322
--- /dev/null
+++ b/checkpoints/model_weights_000045842432.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e0a62b9f18b801474e1c478969074d2990e9eac81a55d6ce91106cb292ce208
+size 225208789
diff --git a/checkpoints/model_weights_000049152000.pt b/checkpoints/model_weights_000049152000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a9cb86271e8b964f3983f0e600fc456d3dde57a5
--- /dev/null
+++ b/checkpoints/model_weights_000049152000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ca71c4af64c79e76cbe567bc8a1fca137685d6382157e6a550e515d9db708b2
+size 225208789
diff --git a/checkpoints/model_weights_000050397184.pt b/checkpoints/model_weights_000050397184.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c00800f025709c2f7dfd4b18f012f0cff2d719e3
--- /dev/null
+++ b/checkpoints/model_weights_000050397184.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ee11989b539ebd83148997360f2105550c3435ab86c5cc4431bcdb19e791b88
+size 225208789
diff --git a/checkpoints/model_weights_000055443456.pt b/checkpoints/model_weights_000055443456.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c6d99f626984bf977aa0902dd947db19c39fa72
--- /dev/null
+++ b/checkpoints/model_weights_000055443456.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:174214bb78b2ccca2e7ea37f91c672c22e1ed9f1041301547f401aa18b613ffb
+size 225208789
diff --git a/checkpoints/model_weights_000061014016.pt b/checkpoints/model_weights_000061014016.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4732c8a0539b04940ec3c96f023bce367a5e6bac
--- /dev/null
+++ b/checkpoints/model_weights_000061014016.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d7602b1d8462d6c1d097ca29868cbaf99249fdaadeed25ea2994dc9b971b15
+size 225208789
diff --git a/checkpoints/model_weights_000065536000.pt b/checkpoints/model_weights_000065536000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c5a56cfff4e82428f7a04e56d41d8c93810f696
--- /dev/null
+++ b/checkpoints/model_weights_000065536000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce23154c56c8ed24547cacedc8bed41e94e4e8964d839faf4d49cb8a99d55c9
+size 225208789
diff --git a/checkpoints/model_weights_000067108864.pt b/checkpoints/model_weights_000067108864.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b19d0563b77d81c5eeea32f3d57eb3a3de3c095d
--- /dev/null
+++ b/checkpoints/model_weights_000067108864.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:985a5d54b5e39965a0b51a00ef7a8d171544422f437caa8270cd33685dcf74f4
+size 225208789
diff --git a/checkpoints/model_weights_000073826304.pt b/checkpoints/model_weights_000073826304.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10045984d8fa80a06c4c776510c2c6581cf4bd54
--- /dev/null
+++ b/checkpoints/model_weights_000073826304.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a0bf97ed206ddbcfbf0a0eb29f3e7035ee7e8690c462fc5e342c2f4dcb06a0e
+size 225208789
diff --git a/checkpoints/model_weights_000081199104.pt b/checkpoints/model_weights_000081199104.pt
new file mode 100644
index 0000000000000000000000000000000000000000..190bcf9807ec79a1693853208ec6b7af7e3a706b
--- /dev/null
+++ b/checkpoints/model_weights_000081199104.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e59f46d1ad9953a717d04a22c3731a1ef5fae06b3250e791a8e16c785a01647
+size 225208789
diff --git a/checkpoints/model_weights_000081920000.pt b/checkpoints/model_weights_000081920000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba7fde0018a38b1844d828d525769cce39b0f885
--- /dev/null
+++ b/checkpoints/model_weights_000081920000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfba6d791606d48dfb6c341cf3d9ac3798d1abfbb4829fcfcc40d92611525d5c
+size 225208789
diff --git a/checkpoints/model_weights_000089325568.pt b/checkpoints/model_weights_000089325568.pt
new file mode 100644
index 0000000000000000000000000000000000000000..900b03d55088d16be2049a8b0e4bb21ca9f218a0
--- /dev/null
+++ b/checkpoints/model_weights_000089325568.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27d0991b4194373b247a06740cf094295ac5c5f2b811f5997098b5ac21dbe0ca
+size 225208789
diff --git a/checkpoints/model_weights_000098271232.pt b/checkpoints/model_weights_000098271232.pt
new file mode 100644
index 0000000000000000000000000000000000000000..18e7c88503e6630412809819cbd3d1e0811433e6
--- /dev/null
+++ b/checkpoints/model_weights_000098271232.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71d4a2c29060677f46c99bc06dcba3289ee0edd6cdfd4c164c5aa08e3475ca3f
+size 225208789
diff --git a/checkpoints/model_weights_000098304000.pt b/checkpoints/model_weights_000098304000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ead8a0ce32224bbad5e77db06c21de4cbb28fe4c
--- /dev/null
+++ b/checkpoints/model_weights_000098304000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7eed2d6d4bb2c5a0f308e952bdc5f060e76a0497f80477b98ab5cca755393b
+size 225208789
diff --git a/checkpoints/model_weights_000108068864.pt b/checkpoints/model_weights_000108068864.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b42525c7cc83648bb780ed72171c289ed1611c04
--- /dev/null
+++ b/checkpoints/model_weights_000108068864.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8ddffa2df45c15e5b546c2282a90c7bf45f6e391a0ffae28d7b6881046d0951
+size 225208789
diff --git a/checkpoints/model_weights_000114688000.pt b/checkpoints/model_weights_000114688000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb67764b7f2e3e892058815b8abac68ba1ad4093
--- /dev/null
+++ b/checkpoints/model_weights_000114688000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b777b26d72245bab98e22e1887f50a9cadccbbaea471ca315d2fb5aa17f7190
+size 225208789
diff --git a/checkpoints/model_weights_000118882304.pt b/checkpoints/model_weights_000118882304.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1797727eb326d41f988fea0c9c075467f71c4cfd
--- /dev/null
+++ b/checkpoints/model_weights_000118882304.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35ce4ae05ee1b516f5b5c7be48f333fae3c09a96aa26c166379fe4aa07720320
+size 225208789
diff --git a/checkpoints/model_weights_000130777088.pt b/checkpoints/model_weights_000130777088.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4be2274fd0f54b3185c0092ea54b4eecaa9d53f5
--- /dev/null
+++ b/checkpoints/model_weights_000130777088.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cb0d76e11c7f246eb8a0105208a729581bff416dc60cf7c5044cba799c6bac8
+size 225208789
diff --git a/checkpoints/model_weights_000131072000.pt b/checkpoints/model_weights_000131072000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e88b6e1e063838c1ebb8df3e6306d1d0493a252a
--- /dev/null
+++ b/checkpoints/model_weights_000131072000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6164b9eb295e1a0fc501ce2ef16dbc78a7727574d80d482f3ac293eca4bcc705
+size 225208789
diff --git a/checkpoints/model_weights_000143851520.pt b/checkpoints/model_weights_000143851520.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86cf75e551c0bfa027a35b9fad9dfa11e4ae1376
--- /dev/null
+++ b/checkpoints/model_weights_000143851520.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ab2fda7df70de34ed79cc65cab8ab55457c2d980c722872b95d6076cabc658
+size 225208789
diff --git a/checkpoints/model_weights_000147456000.pt b/checkpoints/model_weights_000147456000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25b24e0c05eb2a83e97423e4db9fa1bb0225ec1e
--- /dev/null
+++ b/checkpoints/model_weights_000147456000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59a93e1093e64a890891d1174583f31fea6f92ecca3f8051b4c7e610571ecf26
+size 225208789
diff --git a/checkpoints/model_weights_000158269440.pt b/checkpoints/model_weights_000158269440.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d528e252225f84c9a01b9393cf17a87b9cf2e2c4
--- /dev/null
+++ b/checkpoints/model_weights_000158269440.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcdef6dd0f8473d9f3551af7170568ad4eb29ce8a529b089f74d8e9df82ac98b
+size 225208789
diff --git a/checkpoints/model_weights_000163840000.pt b/checkpoints/model_weights_000163840000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9656e51010c20045f000d25aa0566e2628b6e6df
--- /dev/null
+++ b/checkpoints/model_weights_000163840000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:375b3840ad72ddb709a33e57a9697e4dae456f39552d6fc307a94fe51a873e84
+size 225208789
diff --git a/checkpoints/model_weights_000174096384.pt b/checkpoints/model_weights_000174096384.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8f0c3ddfd84fc9e59adea0e86ca00b61a749058
--- /dev/null
+++ b/checkpoints/model_weights_000174096384.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a3c5e498b7ed71e355524819efe3d8710e6878811a0f5f8820cecd64a66d70a
+size 225208789
diff --git a/checkpoints/model_weights_000180224000.pt b/checkpoints/model_weights_000180224000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d70719a35571d3339c91fa955b23a887556f9f9
--- /dev/null
+++ b/checkpoints/model_weights_000180224000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7397d5f60c565b00ad182895c65748711cf46dc15f386e7d813fa358a07b26fc
+size 225208789
diff --git a/checkpoints/model_weights_000191496192.pt b/checkpoints/model_weights_000191496192.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a1656a185c50f53851467b68ea2c078e9a6b25b7
--- /dev/null
+++ b/checkpoints/model_weights_000191496192.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8c26feb8e9e89a118537270ab0deb166f8a7c17e11edf9ecdfb7690567ed28c
+size 225208789
diff --git a/checkpoints/model_weights_000196608000.pt b/checkpoints/model_weights_000196608000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88a79d5a947e252d1a81b85ad06a79c4be252cba
--- /dev/null
+++ b/checkpoints/model_weights_000196608000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4549d2c35a6c8088914c9c7cd7b45662613862114707c235741d80bb7e49083e
+size 225208789
diff --git a/checkpoints/model_weights_000196706304.pt b/checkpoints/model_weights_000196706304.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1493f365ba812cfbdf76e333f446c42ee304e0ec
--- /dev/null
+++ b/checkpoints/model_weights_000196706304.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c068222bd20a7fa264d0882b1b515dda5a21c1f544149c9ceb8680cdcd470c9d
+size 225208789
diff --git a/checkpoints/model_weights_000197361664.pt b/checkpoints/model_weights_000197361664.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7130b4885bbfc1d620909809b0f21575fefa6938
--- /dev/null
+++ b/checkpoints/model_weights_000197361664.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be50b0e92e698cbf869f118d3f2040f5d925ae9127efac07eeb9b90bd889e019
+size 225208789
diff --git a/checkpoints/model_weights_000198017024.pt b/checkpoints/model_weights_000198017024.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb82b610e3cc7668390aef0e1eb59b428b8548d8
--- /dev/null
+++ b/checkpoints/model_weights_000198017024.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9fcfd1e8223b672ee8743508f7819f631770558d40526be8bc040d8ea9291e8
+size 225208789
diff --git a/checkpoints/model_weights_000198672384.pt b/checkpoints/model_weights_000198672384.pt
new file mode 100644
index 0000000000000000000000000000000000000000..212ce0a58118c1a4b9fb2b5c4de37026d8f42c48
--- /dev/null
+++ b/checkpoints/model_weights_000198672384.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c228708334f6baec2d52ce1a18094318d1f449a24ca4169cfadb61a0b94aca55
+size 225208789
diff --git a/checkpoints/model_weights_000199327744.pt b/checkpoints/model_weights_000199327744.pt
new file mode 100644
index 0000000000000000000000000000000000000000..573e9c8569c1807e80e3a5e55036c747ec3d61e6
--- /dev/null
+++ b/checkpoints/model_weights_000199327744.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d4cbe27d167972632f3b34ee61b4b2c5712adbd5102ee9b227fb49f47d6eaa0
+size 225208789
diff --git a/checkpoints/model_weights_000199950336.pt b/checkpoints/model_weights_000199950336.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94dc8929e37c0a0675f05f2e9407332ec156dbac
--- /dev/null
+++ b/checkpoints/model_weights_000199950336.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1a0d2b9c337a5b71374dde4360ef74734a1547f3b568603dee578fa5789338a
+size 225208789
diff --git a/config.toml b/config.toml
new file mode 100644
index 0000000000000000000000000000000000000000..4309a45c9524514790d80fa05c3e4f80023460eb
--- /dev/null
+++ b/config.toml
@@ -0,0 +1,32 @@
+model_name = "c4_SeqUnmodified_NonSeqObserved_L2"
+n_layers = 2
+d_model = 512
+d_mlp = 2048
+d_head = 64
+n_heads = 8
+attn_only = false
+layer_norm_eps = 1e-05
+init_range = 0.02
+n_ctx = 1024
+d_vocab = 48262
+dataset_name = "eoinf/c4_SeqUnmodified_NonSeqObserved_L2"
+tokenizer_name = "NeelNanda/gpt-neox-tokenizer-digits"
+seed = 10
+device = "cuda"
+use_bfloat16_matmul = false
+batch_size_per_device = 32
+n_devices = 1
+batches_per_step = 1
+max_tokens = 200000000
+lr_hidden = 0.002
+lr_vector = 0.001
+lr_schedule = "constant_with_warmup"
+warmup_tokens = 30000000
+weight_decay = 0.05
+grad_norm_clip = 1.0
+train_loss_moving_average_beta = 0.99
+log_interval = 25
+save_checkpoints = true
+checkpoint_interval = 500
+checkpoint_interval_ratio = 1.10
+save_log_checkpoints = true
diff --git a/latest_checkpoint.pt b/latest_checkpoint.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c418864f917c6d908fcc6374e2706954f3693e3c
--- /dev/null
+++ b/latest_checkpoint.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef02ea209551ac937c7d5789dddfb9ee6568bb877865a988acaf4db2e1c5d9a4
+size 225208311
diff --git a/latest_metadata.json b/latest_metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..6650c37319f0a1f0bd58ee3d44267d41f87216a4
--- /dev/null
+++ b/latest_metadata.json
@@ -0,0 +1 @@
+{"step": 6102, "tokens_seen": 199950336, "config": {"model_name": "c4_SeqUnmodified_NonSeqObserved_L2", "n_layers": 2, "d_model": 512, "d_mlp": 2048, "d_head": 64, "n_heads": 8, "attn_only": false, "layer_norm_eps": 1e-05, "init_range": 0.02, "n_ctx": 1024, "d_vocab": 48262, "dataset_name": "eoinf/c4_SeqUnmodified_NonSeqObserved_L2", "tokenizer_name": "NeelNanda/gpt-neox-tokenizer-digits", "seed": 10, "device": "cuda", "use_bfloat16_matmul": false, "batch_size_per_device": 32, "n_devices": 1, "batches_per_step": 1, "max_tokens": 200000000, "lr_hidden": 0.002, "lr_vector": 0.001, "lr_schedule": "constant_with_warmup", "warmup_tokens": 30000000, "weight_decay": 0.05, "grad_norm_clip": 1.0, "train_loss_moving_average_beta": 0.99, "log_interval": 25, "save_checkpoints": true, "checkpoint_interval": 500, "checkpoint_interval_ratio": 1.1, "save_log_checkpoints": true, "use_wandb": true, "batch_size": 32, "tokens_per_step": 32768, "warmup_steps": 915, "max_steps": 6103}, "train_loss_ewma": 6.176267960488811}
\ No newline at end of file
diff --git a/latest_optimizer.pt b/latest_optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8319ad31a5b03f9ff9fbf9d13f7f66052d16ed66
--- /dev/null
+++ b/latest_optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e17f572c0629504e83b25286d0070d2729654edf459eae2c479d94bfa207a7e
+size 450422547
diff --git a/run.sh b/run.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5e78fbd811f1b0ca25e3c932a4c6ee3c0fa5e675
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Check if "restart" argument is passed to force normal training
+if [ "$1" = "restart" ]; then
+    echo "Force restart: Running normal training ..."
+    python -c "
+import os
+from toy_models.models.trainer import train_transformer_from_config
+current_dir = os.getcwd()
+train_transformer_from_config('config.toml', current_dir)
+"
+else
+    # Check for checkpoints and run appropriate training
+    python -c "
+import os
+from pathlib import Path
+from toy_models.models.trainer import train_transformer_from_config, restart_from_checkpoint
+current_dir = os.getcwd()
+# Check if checkpoints directory exists and has .pt files
+latest_checkpoint = Path('latest_checkpoint.pt')
+if latest_checkpoint.exists():
+    print(f'Found checkpoint: {latest_checkpoint}. Restarting from checkpoint...')
+    restart_from_checkpoint(current_dir)
+else:
+    print('Starting training from beginning ...')
+    train_transformer_from_config(current_dir)
+"
+fi
\ No newline at end of file
diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c1c6beec2e82bd8bb1b3f38d57d48ece39d02d05
--- /dev/null
+++ b/wandb/debug-internal.log
@@ -0,0 +1,12 @@
+{"time":"2025-09-26T23:35:54.403287329Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-26T23:35:54.609764327Z","level":"INFO","msg":"stream: created new stream","id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609806816Z","level":"INFO","msg":"stream: started","id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609823798Z","level":"INFO","msg":"writer: started","stream_id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609841971Z","level":"INFO","msg":"handler: started","stream_id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609869143Z","level":"INFO","msg":"sender: started","stream_id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:08.838815981Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":0.022747613}],"total_operations":1}}
+{"time":"2025-09-27T00:44:09.172329967Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-27T00:44:09.297375375Z","level":"INFO","msg":"stream: closing","id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:09.297412196Z","level":"INFO","msg":"handler: closed","stream_id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:09.297486349Z","level":"INFO","msg":"sender: closed","stream_id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:09.297492707Z","level":"INFO","msg":"stream: closed","id":"wxnxan9r"}
diff --git a/wandb/debug.log b/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..4676117587174aa747f6212dba2c6ddca74aa7fa
--- /dev/null
+++ b/wandb/debug.log
@@ -0,0 +1,26 @@
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Configure stats pid to 540
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2/wandb/settings
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2/wandb/run-20250926_233554-wxnxan9r/logs/debug.log
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2/wandb/run-20250926_233554-wxnxan9r/logs/debug-internal.log
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:init():813] calling init triggers
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'c4_SeqUnmodified_NonSeqObserved_L2', 'n_layers': 2, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 48262, 'dataset_name': 'eoinf/c4_SeqUnmodified_NonSeqObserved_L2', 'tokenizer_name': 'NeelNanda/gpt-neox-tokenizer-digits', 'seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 32, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.002, 'lr_vector': 0.001, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 32, 'tokens_per_step': 32768, 'warmup_steps': 915, 'max_steps': 6103, '_wandb': {}}
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:init():854] starting backend
+2025-09-26 23:35:54,387 INFO    MainThread:540 [wandb_init.py:init():857] sending inform_init request
+2025-09-26 23:35:54,398 INFO    MainThread:540 [wandb_init.py:init():865] backend started and connected
+2025-09-26 23:35:54,399 INFO    MainThread:540 [wandb_init.py:init():936] updated telemetry
+2025-09-26 23:35:54,439 INFO    MainThread:540 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-26 23:35:54,734 INFO    MainThread:540 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_console_start():2506] atexit reg
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_redirect():2446] Redirects installed.
+2025-09-26 23:35:55,252 INFO    MainThread:540 [wandb_init.py:init():1049] run started, returning control to user process
+2025-09-27 00:44:08,798 INFO    MainThread:540 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/wxnxan9r
+2025-09-27 00:44:08,800 INFO    MainThread:540 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0
+2025-09-27 00:44:08,800 INFO    MainThread:540 [wandb_run.py:_restore():2453] restore
+2025-09-27 00:44:08,800 INFO    MainThread:540 [wandb_run.py:_restore():2459] restore done
+2025-09-27 00:44:09,296 INFO    MainThread:540 [wandb_run.py:_footer_sync_info():3867] logging synced files
diff --git a/wandb/run-20250926_233554-wxnxan9r/files/config.yaml b/wandb/run-20250926_233554-wxnxan9r/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a96d0ca620f9fe129cf07d4bdc0ad717e21b8a16
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/files/config.yaml
@@ -0,0 +1,134 @@
+_wandb:
+    value:
+        cli_version: 0.21.4
+        e:
+            s2y1kfnn063xv2a8fa1iwgztn27t3afe:
+                cpu_count: 8
+                cpu_count_logical: 8
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "262240792576"
+                        used: "154771664896"
+                email: tzfof8@gmail.com
+                executable: /notebooks/toy_models/.toy_models_env/bin/python
+                git:
+                    commit: 750b6c98784324c15d9d813c9e365e5ccfa8b0ab
+                    remote: https://github.com/jgroh3/toy_models.git
+                gpu: NVIDIA RTX A6000
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 10752
+                      memoryTotal: "51527024640"
+                      name: NVIDIA RTX A6000
+                      uuid: GPU-7b3bcd99-03fe-0e7c-86cb-d143c0cb514f
+                host: nq6cv1dtxr
+                memory:
+                    total: "47332843520"
+                os: Linux-5.19.0-45-generic-x86_64-with-glibc2.35
+                program: <python with no main file>
+                python: CPython 3.11.7
+                root: /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2
+                startedAt: "2025-09-26T23:35:54.126498Z"
+                writerId: s2y1kfnn063xv2a8fa1iwgztn27t3afe
+        m: []
+        python_version: 3.11.7
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.11.7
+            "5": 0.21.4
+            "6": 4.56.1
+            "12": 0.21.4
+            "13": linux-x86_64
+attn_only:
+    value: false
+batch_size:
+    value: 32
+batch_size_per_device:
+    value: 32
+batches_per_step:
+    value: 1
+checkpoint_interval:
+    value: 500
+checkpoint_interval_ratio:
+    value: 1.1
+d_head:
+    value: 64
+d_mlp:
+    value: 2048
+d_model:
+    value: 512
+d_vocab:
+    value: 48262
+dataset_name:
+    value: eoinf/c4_SeqUnmodified_NonSeqObserved_L2
+device:
+    value: cuda
+grad_norm_clip:
+    value: 1
+init_range:
+    value: 0.02
+layer_norm_eps:
+    value: 1e-05
+log_interval:
+    value: 25
+lr_hidden:
+    value: 0.002
+lr_schedule:
+    value: constant_with_warmup
+lr_vector:
+    value: 0.001
+max_steps:
+    value: 6103
+max_tokens:
+    value: 200000000
+model_name:
+    value: c4_SeqUnmodified_NonSeqObserved_L2
+n_ctx:
+    value: 1024
+n_devices:
+    value: 1
+n_heads:
+    value: 8
+n_layers:
+    value: 2
+save_checkpoints:
+    value: true
+save_log_checkpoints:
+    value: true
+seed:
+    value: 10
+tokenizer_name:
+    value: NeelNanda/gpt-neox-tokenizer-digits
+tokens_per_step:
+    value: 32768
+train_loss_moving_average_beta:
+    value: 0.99
+use_bfloat16_matmul:
+    value: false
+use_wandb:
+    value: true
+warmup_steps:
+    value: 915
+warmup_tokens:
+    value: 30000000
+weight_decay:
+    value: 0.05
diff --git a/wandb/run-20250926_233554-wxnxan9r/files/output.log b/wandb/run-20250926_233554-wxnxan9r/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..6660c0d4d279936743695891d591bb65ab1ed815
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/files/output.log
@@ -0,0 +1,252 @@
+Training on cuda
+Model: 2L, 512d, 8h
+Max steps: 6,103, Max tokens: 200,000,000
+Warmup steps: 915, Warmup tokens: 30,000,000
+Batch size per device: 32
+Context length: 1024
+Learning rates - Hidden: 0.002, Vector: 0.001
+                                                                                                                                                                                                       
+Step 25 | Tokens: 819,200 | Train Loss EWMA: 10.7911 | Learning Rate: 0.000055 | Progress: 0.00410
+Step 50 | Tokens: 1,638,400 | Train Loss EWMA: 10.5605 | Learning Rate: 0.000109 | Progress: 0.00819
+Step 75 | Tokens: 2,457,600 | Train Loss EWMA: 10.2011 | Learning Rate: 0.000164 | Progress: 0.01229
+Step 100 | Tokens: 3,276,800 | Train Loss EWMA: 9.7337 | Learning Rate: 0.000219 | Progress: 0.01638
+Step 125 | Tokens: 4,096,000 | Train Loss EWMA: 9.2655 | Learning Rate: 0.000273 | Progress: 0.02048
+Step 150 | Tokens: 4,915,200 | Train Loss EWMA: 8.8717 | Learning Rate: 0.000328 | Progress: 0.02458
+Step 175 | Tokens: 5,734,400 | Train Loss EWMA: 8.5568 | Learning Rate: 0.000383 | Progress: 0.02867
+Step 200 | Tokens: 6,553,600 | Train Loss EWMA: 8.2988 | Learning Rate: 0.000437 | Progress: 0.03277
+Step 225 | Tokens: 7,372,800 | Train Loss EWMA: 8.0967 | Learning Rate: 0.000492 | Progress: 0.03686
+Step 250 | Tokens: 8,192,000 | Train Loss EWMA: 7.9267 | Learning Rate: 0.000546 | Progress: 0.04096
+Step 275 | Tokens: 9,011,200 | Train Loss EWMA: 7.7916 | Learning Rate: 0.000601 | Progress: 0.04506
+Step 300 | Tokens: 9,830,400 | Train Loss EWMA: 7.6823 | Learning Rate: 0.000656 | Progress: 0.04915
+Step 325 | Tokens: 10,649,600 | Train Loss EWMA: 7.5980 | Learning Rate: 0.000710 | Progress: 0.05325
+Step 350 | Tokens: 11,468,800 | Train Loss EWMA: 7.5168 | Learning Rate: 0.000765 | Progress: 0.05734
+Step 375 | Tokens: 12,288,000 | Train Loss EWMA: 7.4579 | Learning Rate: 0.000820 | Progress: 0.06144
+Step 400 | Tokens: 13,107,200 | Train Loss EWMA: 7.4102 | Learning Rate: 0.000874 | Progress: 0.06554
+Step 425 | Tokens: 13,926,400 | Train Loss EWMA: 7.3693 | Learning Rate: 0.000929 | Progress: 0.06963
+Step 450 | Tokens: 14,745,600 | Train Loss EWMA: 7.3353 | Learning Rate: 0.000984 | Progress: 0.07373
+Step 475 | Tokens: 15,564,800 | Train Loss EWMA: 7.3091 | Learning Rate: 0.001038 | Progress: 0.07782
+Step 500 | Tokens: 16,384,000 | Train Loss EWMA: 7.2943 | Learning Rate: 0.001093 | Progress: 0.08192
+Step 525 | Tokens: 17,203,200 | Train Loss EWMA: 7.2715 | Learning Rate: 0.001148 | Progress: 0.08602
+Step 550 | Tokens: 18,022,400 | Train Loss EWMA: 7.2577 | Learning Rate: 0.001202 | Progress: 0.09011
+Step 575 | Tokens: 18,841,600 | Train Loss EWMA: 7.2412 | Learning Rate: 0.001257 | Progress: 0.09421
+Step 600 | Tokens: 19,660,800 | Train Loss EWMA: 7.2280 | Learning Rate: 0.001311 | Progress: 0.09830
+Step 625 | Tokens: 20,480,000 | Train Loss EWMA: 7.2117 | Learning Rate: 0.001366 | Progress: 0.10240
+Step 650 | Tokens: 21,299,200 | Train Loss EWMA: 7.2047 | Learning Rate: 0.001421 | Progress: 0.10650
+Step 675 | Tokens: 22,118,400 | Train Loss EWMA: 7.1977 | Learning Rate: 0.001475 | Progress: 0.11059
+Step 700 | Tokens: 22,937,600 | Train Loss EWMA: 7.1895 | Learning Rate: 0.001530 | Progress: 0.11469
+Step 725 | Tokens: 23,756,800 | Train Loss EWMA: 7.1763 | Learning Rate: 0.001585 | Progress: 0.11878
+Step 750 | Tokens: 24,576,000 | Train Loss EWMA: 7.1682 | Learning Rate: 0.001639 | Progress: 0.12288
+Step 775 | Tokens: 25,395,200 | Train Loss EWMA: 7.1627 | Learning Rate: 0.001694 | Progress: 0.12698
+Step 800 | Tokens: 26,214,400 | Train Loss EWMA: 7.1597 | Learning Rate: 0.001749 | Progress: 0.13107
+Step 825 | Tokens: 27,033,600 | Train Loss EWMA: 7.1534 | Learning Rate: 0.001803 | Progress: 0.13517
+Step 850 | Tokens: 27,852,800 | Train Loss EWMA: 7.1448 | Learning Rate: 0.001858 | Progress: 0.13926
+Step 875 | Tokens: 28,672,000 | Train Loss EWMA: 7.1364 | Learning Rate: 0.001913 | Progress: 0.14336
+Step 900 | Tokens: 29,491,200 | Train Loss EWMA: 7.1388 | Learning Rate: 0.001967 | Progress: 0.14746
+Step 925 | Tokens: 30,310,400 | Train Loss EWMA: 7.1305 | Learning Rate: 0.002000 | Progress: 0.15155
+Step 950 | Tokens: 31,129,600 | Train Loss EWMA: 7.1300 | Learning Rate: 0.002000 | Progress: 0.15565
+Step 975 | Tokens: 31,948,800 | Train Loss EWMA: 7.1198 | Learning Rate: 0.002000 | Progress: 0.15974
+Step 1,000 | Tokens: 32,768,000 | Train Loss EWMA: 7.1150 | Learning Rate: 0.002000 | Progress: 0.16384
+Step 1,025 | Tokens: 33,587,200 | Train Loss EWMA: 7.1156 | Learning Rate: 0.002000 | Progress: 0.16794
+Step 1,050 | Tokens: 34,406,400 | Train Loss EWMA: 7.1157 | Learning Rate: 0.002000 | Progress: 0.17203
+Step 1,075 | Tokens: 35,225,600 | Train Loss EWMA: 7.1010 | Learning Rate: 0.002000 | Progress: 0.17613
+Step 1,100 | Tokens: 36,044,800 | Train Loss EWMA: 7.1039 | Learning Rate: 0.002000 | Progress: 0.18022
+Step 1,125 | Tokens: 36,864,000 | Train Loss EWMA: 7.0923 | Learning Rate: 0.002000 | Progress: 0.18432
+Step 1,150 | Tokens: 37,683,200 | Train Loss EWMA: 7.0841 | Learning Rate: 0.002000 | Progress: 0.18842
+Step 1,175 | Tokens: 38,502,400 | Train Loss EWMA: 7.0869 | Learning Rate: 0.002000 | Progress: 0.19251
+Step 1,200 | Tokens: 39,321,600 | Train Loss EWMA: 7.0724 | Learning Rate: 0.002000 | Progress: 0.19661
+Step 1,225 | Tokens: 40,140,800 | Train Loss EWMA: 7.0687 | Learning Rate: 0.002000 | Progress: 0.20070
+Step 1,250 | Tokens: 40,960,000 | Train Loss EWMA: 7.0754 | Learning Rate: 0.002000 | Progress: 0.20480
+Step 1,275 | Tokens: 41,779,200 | Train Loss EWMA: 7.0726 | Learning Rate: 0.002000 | Progress: 0.20890
+Step 1,300 | Tokens: 42,598,400 | Train Loss EWMA: 7.0637 | Learning Rate: 0.002000 | Progress: 0.21299
+Step 1,325 | Tokens: 43,417,600 | Train Loss EWMA: 7.0666 | Learning Rate: 0.002000 | Progress: 0.21709
+Step 1,350 | Tokens: 44,236,800 | Train Loss EWMA: 7.0640 | Learning Rate: 0.002000 | Progress: 0.22118
+Step 1,375 | Tokens: 45,056,000 | Train Loss EWMA: 7.0636 | Learning Rate: 0.002000 | Progress: 0.22528
+Step 1,400 | Tokens: 45,875,200 | Train Loss EWMA: 7.0465 | Learning Rate: 0.002000 | Progress: 0.22938
+Step 1,425 | Tokens: 46,694,400 | Train Loss EWMA: 7.0362 | Learning Rate: 0.002000 | Progress: 0.23347
+Step 1,450 | Tokens: 47,513,600 | Train Loss EWMA: 7.0361 | Learning Rate: 0.002000 | Progress: 0.23757
+Step 1,475 | Tokens: 48,332,800 | Train Loss EWMA: 7.0320 | Learning Rate: 0.002000 | Progress: 0.24166
+Step 1,500 | Tokens: 49,152,000 | Train Loss EWMA: 7.0259 | Learning Rate: 0.002000 | Progress: 0.24576
+Step 1,525 | Tokens: 49,971,200 | Train Loss EWMA: 7.0206 | Learning Rate: 0.002000 | Progress: 0.24986
+Step 1,550 | Tokens: 50,790,400 | Train Loss EWMA: 7.0183 | Learning Rate: 0.002000 | Progress: 0.25395
+Step 1,575 | Tokens: 51,609,600 | Train Loss EWMA: 7.0248 | Learning Rate: 0.002000 | Progress: 0.25805
+Step 1,600 | Tokens: 52,428,800 | Train Loss EWMA: 7.0183 | Learning Rate: 0.002000 | Progress: 0.26214
+Step 1,625 | Tokens: 53,248,000 | Train Loss EWMA: 7.0139 | Learning Rate: 0.002000 | Progress: 0.26624
+Step 1,650 | Tokens: 54,067,200 | Train Loss EWMA: 7.0173 | Learning Rate: 0.002000 | Progress: 0.27034
+Step 1,675 | Tokens: 54,886,400 | Train Loss EWMA: 7.0162 | Learning Rate: 0.002000 | Progress: 0.27443
+Step 1,700 | Tokens: 55,705,600 | Train Loss EWMA: 7.0148 | Learning Rate: 0.002000 | Progress: 0.27853
+Step 1,725 | Tokens: 56,524,800 | Train Loss EWMA: 7.0101 | Learning Rate: 0.002000 | Progress: 0.28262
+Step 1,750 | Tokens: 57,344,000 | Train Loss EWMA: 7.0120 | Learning Rate: 0.002000 | Progress: 0.28672
+Step 1,775 | Tokens: 58,163,200 | Train Loss EWMA: 7.0042 | Learning Rate: 0.002000 | Progress: 0.29082
+Step 1,800 | Tokens: 58,982,400 | Train Loss EWMA: 7.0011 | Learning Rate: 0.002000 | Progress: 0.29491
+Step 1,825 | Tokens: 59,801,600 | Train Loss EWMA: 6.9960 | Learning Rate: 0.002000 | Progress: 0.29901
+Step 1,850 | Tokens: 60,620,800 | Train Loss EWMA: 6.9959 | Learning Rate: 0.002000 | Progress: 0.30310
+Step 1,875 | Tokens: 61,440,000 | Train Loss EWMA: 6.9888 | Learning Rate: 0.002000 | Progress: 0.30720
+Step 1,900 | Tokens: 62,259,200 | Train Loss EWMA: 6.9791 | Learning Rate: 0.002000 | Progress: 0.31130
+Step 1,925 | Tokens: 63,078,400 | Train Loss EWMA: 6.9755 | Learning Rate: 0.002000 | Progress: 0.31539
+Step 1,950 | Tokens: 63,897,600 | Train Loss EWMA: 6.9823 | Learning Rate: 0.002000 | Progress: 0.31949
+Step 1,975 | Tokens: 64,716,800 | Train Loss EWMA: 6.9766 | Learning Rate: 0.002000 | Progress: 0.32358
+Step 2,000 | Tokens: 65,536,000 | Train Loss EWMA: 6.9717 | Learning Rate: 0.002000 | Progress: 0.32768
+Step 2,025 | Tokens: 66,355,200 | Train Loss EWMA: 6.9659 | Learning Rate: 0.002000 | Progress: 0.33178
+Step 2,050 | Tokens: 67,174,400 | Train Loss EWMA: 6.9587 | Learning Rate: 0.002000 | Progress: 0.33587
+Step 2,075 | Tokens: 67,993,600 | Train Loss EWMA: 6.9434 | Learning Rate: 0.002000 | Progress: 0.33997
+Step 2,100 | Tokens: 68,812,800 | Train Loss EWMA: 6.9425 | Learning Rate: 0.002000 | Progress: 0.34406
+Step 2,125 | Tokens: 69,632,000 | Train Loss EWMA: 6.9440 | Learning Rate: 0.002000 | Progress: 0.34816
+Step 2,150 | Tokens: 70,451,200 | Train Loss EWMA: 6.9553 | Learning Rate: 0.002000 | Progress: 0.35226
+Step 2,175 | Tokens: 71,270,400 | Train Loss EWMA: 6.9483 | Learning Rate: 0.002000 | Progress: 0.35635
+Step 2,200 | Tokens: 72,089,600 | Train Loss EWMA: 6.9529 | Learning Rate: 0.002000 | Progress: 0.36045
+Step 2,225 | Tokens: 72,908,800 | Train Loss EWMA: 6.9471 | Learning Rate: 0.002000 | Progress: 0.36454
+Step 2,250 | Tokens: 73,728,000 | Train Loss EWMA: 6.9408 | Learning Rate: 0.002000 | Progress: 0.36864
+Step 2,275 | Tokens: 74,547,200 | Train Loss EWMA: 6.9411 | Learning Rate: 0.002000 | Progress: 0.37274
+Step 2,300 | Tokens: 75,366,400 | Train Loss EWMA: 6.9333 | Learning Rate: 0.002000 | Progress: 0.37683
+Step 2,325 | Tokens: 76,185,600 | Train Loss EWMA: 6.9342 | Learning Rate: 0.002000 | Progress: 0.38093
+Step 2,350 | Tokens: 77,004,800 | Train Loss EWMA: 6.9348 | Learning Rate: 0.002000 | Progress: 0.38502
+Step 2,375 | Tokens: 77,824,000 | Train Loss EWMA: 6.9376 | Learning Rate: 0.002000 | Progress: 0.38912
+Step 2,400 | Tokens: 78,643,200 | Train Loss EWMA: 6.9334 | Learning Rate: 0.002000 | Progress: 0.39322
+Step 2,425 | Tokens: 79,462,400 | Train Loss EWMA: 6.9158 | Learning Rate: 0.002000 | Progress: 0.39731
+Step 2,450 | Tokens: 80,281,600 | Train Loss EWMA: 6.9231 | Learning Rate: 0.002000 | Progress: 0.40141
+Step 2,475 | Tokens: 81,100,800 | Train Loss EWMA: 6.9187 | Learning Rate: 0.002000 | Progress: 0.40550
+Step 2,500 | Tokens: 81,920,000 | Train Loss EWMA: 6.9184 | Learning Rate: 0.002000 | Progress: 0.40960
+Step 2,525 | Tokens: 82,739,200 | Train Loss EWMA: 6.9209 | Learning Rate: 0.002000 | Progress: 0.41370
+Step 2,550 | Tokens: 83,558,400 | Train Loss EWMA: 6.9055 | Learning Rate: 0.002000 | Progress: 0.41779
+Step 2,575 | Tokens: 84,377,600 | Train Loss EWMA: 6.8953 | Learning Rate: 0.002000 | Progress: 0.42189
+Step 2,600 | Tokens: 85,196,800 | Train Loss EWMA: 6.8903 | Learning Rate: 0.002000 | Progress: 0.42598
+Step 2,625 | Tokens: 86,016,000 | Train Loss EWMA: 6.8984 | Learning Rate: 0.002000 | Progress: 0.43008
+Step 2,650 | Tokens: 86,835,200 | Train Loss EWMA: 6.8936 | Learning Rate: 0.002000 | Progress: 0.43418
+Step 2,675 | Tokens: 87,654,400 | Train Loss EWMA: 6.8964 | Learning Rate: 0.002000 | Progress: 0.43827
+Step 2,700 | Tokens: 88,473,600 | Train Loss EWMA: 6.8895 | Learning Rate: 0.002000 | Progress: 0.44237
+Step 2,725 | Tokens: 89,292,800 | Train Loss EWMA: 6.8864 | Learning Rate: 0.002000 | Progress: 0.44646
+Step 2,750 | Tokens: 90,112,000 | Train Loss EWMA: 6.8801 | Learning Rate: 0.002000 | Progress: 0.45056
+Step 2,775 | Tokens: 90,931,200 | Train Loss EWMA: 6.8829 | Learning Rate: 0.002000 | Progress: 0.45466
+Step 2,800 | Tokens: 91,750,400 | Train Loss EWMA: 6.8805 | Learning Rate: 0.002000 | Progress: 0.45875
+Step 2,825 | Tokens: 92,569,600 | Train Loss EWMA: 6.8764 | Learning Rate: 0.002000 | Progress: 0.46285
+Step 2,850 | Tokens: 93,388,800 | Train Loss EWMA: 6.8774 | Learning Rate: 0.002000 | Progress: 0.46694
+Step 2,875 | Tokens: 94,208,000 | Train Loss EWMA: 6.8724 | Learning Rate: 0.002000 | Progress: 0.47104
+Step 2,900 | Tokens: 95,027,200 | Train Loss EWMA: 6.8846 | Learning Rate: 0.002000 | Progress: 0.47514
+Step 2,925 | Tokens: 95,846,400 | Train Loss EWMA: 6.8711 | Learning Rate: 0.002000 | Progress: 0.47923
+Step 2,950 | Tokens: 96,665,600 | Train Loss EWMA: 6.8553 | Learning Rate: 0.002000 | Progress: 0.48333
+Step 2,975 | Tokens: 97,484,800 | Train Loss EWMA: 6.8456 | Learning Rate: 0.002000 | Progress: 0.48742
+Step 3,000 | Tokens: 98,304,000 | Train Loss EWMA: 6.8362 | Learning Rate: 0.002000 | Progress: 0.49152
+Step 3,025 | Tokens: 99,123,200 | Train Loss EWMA: 6.8301 | Learning Rate: 0.002000 | Progress: 0.49562
+Step 3,050 | Tokens: 99,942,400 | Train Loss EWMA: 6.8217 | Learning Rate: 0.002000 | Progress: 0.49971
+Step 3,075 | Tokens: 100,761,600 | Train Loss EWMA: 6.8072 | Learning Rate: 0.002000 | Progress: 0.50381
+Step 3,100 | Tokens: 101,580,800 | Train Loss EWMA: 6.8093 | Learning Rate: 0.002000 | Progress: 0.50790
+Step 3,125 | Tokens: 102,400,000 | Train Loss EWMA: 6.8064 | Learning Rate: 0.002000 | Progress: 0.51200
+Step 3,150 | Tokens: 103,219,200 | Train Loss EWMA: 6.7882 | Learning Rate: 0.002000 | Progress: 0.51610
+Step 3,175 | Tokens: 104,038,400 | Train Loss EWMA: 6.7519 | Learning Rate: 0.002000 | Progress: 0.52019
+Step 3,200 | Tokens: 104,857,600 | Train Loss EWMA: 6.7217 | Learning Rate: 0.002000 | Progress: 0.52429
+Step 3,225 | Tokens: 105,676,800 | Train Loss EWMA: 6.6855 | Learning Rate: 0.002000 | Progress: 0.52838
+Step 3,250 | Tokens: 106,496,000 | Train Loss EWMA: 6.6634 | Learning Rate: 0.002000 | Progress: 0.53248
+Step 3,275 | Tokens: 107,315,200 | Train Loss EWMA: 6.6432 | Learning Rate: 0.002000 | Progress: 0.53658
+Step 3,300 | Tokens: 108,134,400 | Train Loss EWMA: 6.6225 | Learning Rate: 0.002000 | Progress: 0.54067
+Step 3,325 | Tokens: 108,953,600 | Train Loss EWMA: 6.6048 | Learning Rate: 0.002000 | Progress: 0.54477
+Step 3,350 | Tokens: 109,772,800 | Train Loss EWMA: 6.5859 | Learning Rate: 0.002000 | Progress: 0.54886
+Step 3,375 | Tokens: 110,592,000 | Train Loss EWMA: 6.5599 | Learning Rate: 0.002000 | Progress: 0.55296
+Step 3,400 | Tokens: 111,411,200 | Train Loss EWMA: 6.5477 | Learning Rate: 0.002000 | Progress: 0.55706
+Step 3,425 | Tokens: 112,230,400 | Train Loss EWMA: 6.5316 | Learning Rate: 0.002000 | Progress: 0.56115
+Step 3,450 | Tokens: 113,049,600 | Train Loss EWMA: 6.5159 | Learning Rate: 0.002000 | Progress: 0.56525
+Step 3,475 | Tokens: 113,868,800 | Train Loss EWMA: 6.4950 | Learning Rate: 0.002000 | Progress: 0.56934
+Step 3,500 | Tokens: 114,688,000 | Train Loss EWMA: 6.4846 | Learning Rate: 0.002000 | Progress: 0.57344
+Step 3,525 | Tokens: 115,507,200 | Train Loss EWMA: 6.4704 | Learning Rate: 0.002000 | Progress: 0.57754
+Step 3,550 | Tokens: 116,326,400 | Train Loss EWMA: 6.4645 | Learning Rate: 0.002000 | Progress: 0.58163
+Step 3,575 | Tokens: 117,145,600 | Train Loss EWMA: 6.4409 | Learning Rate: 0.002000 | Progress: 0.58573
+Step 3,600 | Tokens: 117,964,800 | Train Loss EWMA: 6.4362 | Learning Rate: 0.002000 | Progress: 0.58982
+Step 3,625 | Tokens: 118,784,000 | Train Loss EWMA: 6.4130 | Learning Rate: 0.002000 | Progress: 0.59392
+Step 3,650 | Tokens: 119,603,200 | Train Loss EWMA: 6.4117 | Learning Rate: 0.002000 | Progress: 0.59802
+Step 3,675 | Tokens: 120,422,400 | Train Loss EWMA: 6.4082 | Learning Rate: 0.002000 | Progress: 0.60211
+Step 3,700 | Tokens: 121,241,600 | Train Loss EWMA: 6.4148 | Learning Rate: 0.002000 | Progress: 0.60621
+Step 3,725 | Tokens: 122,060,800 | Train Loss EWMA: 6.3968 | Learning Rate: 0.002000 | Progress: 0.61030
+Step 3,750 | Tokens: 122,880,000 | Train Loss EWMA: 6.3967 | Learning Rate: 0.002000 | Progress: 0.61440
+Step 3,775 | Tokens: 123,699,200 | Train Loss EWMA: 6.4010 | Learning Rate: 0.002000 | Progress: 0.61850
+Step 3,800 | Tokens: 124,518,400 | Train Loss EWMA: 6.3857 | Learning Rate: 0.002000 | Progress: 0.62259
+Step 3,825 | Tokens: 125,337,600 | Train Loss EWMA: 6.3913 | Learning Rate: 0.002000 | Progress: 0.62669
+Step 3,850 | Tokens: 126,156,800 | Train Loss EWMA: 6.3707 | Learning Rate: 0.002000 | Progress: 0.63078
+Step 3,875 | Tokens: 126,976,000 | Train Loss EWMA: 6.3699 | Learning Rate: 0.002000 | Progress: 0.63488
+Step 3,900 | Tokens: 127,795,200 | Train Loss EWMA: 6.3717 | Learning Rate: 0.002000 | Progress: 0.63898
+Step 3,925 | Tokens: 128,614,400 | Train Loss EWMA: 6.3648 | Learning Rate: 0.002000 | Progress: 0.64307
+Step 3,950 | Tokens: 129,433,600 | Train Loss EWMA: 6.3521 | Learning Rate: 0.002000 | Progress: 0.64717
+Step 3,975 | Tokens: 130,252,800 | Train Loss EWMA: 6.3690 | Learning Rate: 0.002000 | Progress: 0.65126
+Step 4,000 | Tokens: 131,072,000 | Train Loss EWMA: 6.3539 | Learning Rate: 0.002000 | Progress: 0.65536
+Step 4,025 | Tokens: 131,891,200 | Train Loss EWMA: 6.3688 | Learning Rate: 0.002000 | Progress: 0.65946
+Step 4,050 | Tokens: 132,710,400 | Train Loss EWMA: 6.3602 | Learning Rate: 0.002000 | Progress: 0.66355
+Step 4,075 | Tokens: 133,529,600 | Train Loss EWMA: 6.3549 | Learning Rate: 0.002000 | Progress: 0.66765
+Step 4,100 | Tokens: 134,348,800 | Train Loss EWMA: 6.3441 | Learning Rate: 0.002000 | Progress: 0.67174
+Step 4,125 | Tokens: 135,168,000 | Train Loss EWMA: 6.3373 | Learning Rate: 0.002000 | Progress: 0.67584
+Step 4,150 | Tokens: 135,987,200 | Train Loss EWMA: 6.3295 | Learning Rate: 0.002000 | Progress: 0.67994
+Step 4,175 | Tokens: 136,806,400 | Train Loss EWMA: 6.3292 | Learning Rate: 0.002000 | Progress: 0.68403
+Step 4,200 | Tokens: 137,625,600 | Train Loss EWMA: 6.3202 | Learning Rate: 0.002000 | Progress: 0.68813
+Step 4,225 | Tokens: 138,444,800 | Train Loss EWMA: 6.3166 | Learning Rate: 0.002000 | Progress: 0.69222
+Step 4,250 | Tokens: 139,264,000 | Train Loss EWMA: 6.3044 | Learning Rate: 0.002000 | Progress: 0.69632
+Step 4,275 | Tokens: 140,083,200 | Train Loss EWMA: 6.2953 | Learning Rate: 0.002000 | Progress: 0.70042
+Step 4,300 | Tokens: 140,902,400 | Train Loss EWMA: 6.3030 | Learning Rate: 0.002000 | Progress: 0.70451
+Step 4,325 | Tokens: 141,721,600 | Train Loss EWMA: 6.2983 | Learning Rate: 0.002000 | Progress: 0.70861
+Step 4,350 | Tokens: 142,540,800 | Train Loss EWMA: 6.2952 | Learning Rate: 0.002000 | Progress: 0.71270
+Step 4,375 | Tokens: 143,360,000 | Train Loss EWMA: 6.2888 | Learning Rate: 0.002000 | Progress: 0.71680
+Step 4,400 | Tokens: 144,179,200 | Train Loss EWMA: 6.2678 | Learning Rate: 0.002000 | Progress: 0.72090
+Step 4,425 | Tokens: 144,998,400 | Train Loss EWMA: 6.2657 | Learning Rate: 0.002000 | Progress: 0.72499
+Step 4,450 | Tokens: 145,817,600 | Train Loss EWMA: 6.2665 | Learning Rate: 0.002000 | Progress: 0.72909
+Step 4,475 | Tokens: 146,636,800 | Train Loss EWMA: 6.2443 | Learning Rate: 0.002000 | Progress: 0.73318
+Step 4,500 | Tokens: 147,456,000 | Train Loss EWMA: 6.2544 | Learning Rate: 0.002000 | Progress: 0.73728
+Step 4,525 | Tokens: 148,275,200 | Train Loss EWMA: 6.2537 | Learning Rate: 0.002000 | Progress: 0.74138
+Step 4,550 | Tokens: 149,094,400 | Train Loss EWMA: 6.2592 | Learning Rate: 0.002000 | Progress: 0.74547
+Step 4,575 | Tokens: 149,913,600 | Train Loss EWMA: 6.2520 | Learning Rate: 0.002000 | Progress: 0.74957
+Step 4,600 | Tokens: 150,732,800 | Train Loss EWMA: 6.2675 | Learning Rate: 0.002000 | Progress: 0.75366
+Step 4,625 | Tokens: 151,552,000 | Train Loss EWMA: 6.2630 | Learning Rate: 0.002000 | Progress: 0.75776
+Step 4,650 | Tokens: 152,371,200 | Train Loss EWMA: 6.2729 | Learning Rate: 0.002000 | Progress: 0.76186
+Step 4,675 | Tokens: 153,190,400 | Train Loss EWMA: 6.2661 | Learning Rate: 0.002000 | Progress: 0.76595
+Step 4,700 | Tokens: 154,009,600 | Train Loss EWMA: 6.2680 | Learning Rate: 0.002000 | Progress: 0.77005
+Step 4,725 | Tokens: 154,828,800 | Train Loss EWMA: 6.2551 | Learning Rate: 0.002000 | Progress: 0.77414
+Step 4,750 | Tokens: 155,648,000 | Train Loss EWMA: 6.2619 | Learning Rate: 0.002000 | Progress: 0.77824
+Step 4,775 | Tokens: 156,467,200 | Train Loss EWMA: 6.2706 | Learning Rate: 0.002000 | Progress: 0.78234
+Step 4,800 | Tokens: 157,286,400 | Train Loss EWMA: 6.2507 | Learning Rate: 0.002000 | Progress: 0.78643
+Step 4,825 | Tokens: 158,105,600 | Train Loss EWMA: 6.2459 | Learning Rate: 0.002000 | Progress: 0.79053
+Step 4,850 | Tokens: 158,924,800 | Train Loss EWMA: 6.2658 | Learning Rate: 0.002000 | Progress: 0.79462
+Step 4,875 | Tokens: 159,744,000 | Train Loss EWMA: 6.2806 | Learning Rate: 0.002000 | Progress: 0.79872
+Step 4,900 | Tokens: 160,563,200 | Train Loss EWMA: 6.3054 | Learning Rate: 0.002000 | Progress: 0.80282
+Step 4,925 | Tokens: 161,382,400 | Train Loss EWMA: 6.2931 | Learning Rate: 0.002000 | Progress: 0.80691
+Step 4,950 | Tokens: 162,201,600 | Train Loss EWMA: 6.2841 | Learning Rate: 0.002000 | Progress: 0.81101
+Step 4,975 | Tokens: 163,020,800 | Train Loss EWMA: 6.2778 | Learning Rate: 0.002000 | Progress: 0.81510
+Step 5,000 | Tokens: 163,840,000 | Train Loss EWMA: 6.2684 | Learning Rate: 0.002000 | Progress: 0.81920
+Step 5,025 | Tokens: 164,659,200 | Train Loss EWMA: 6.2640 | Learning Rate: 0.002000 | Progress: 0.82330
+Step 5,050 | Tokens: 165,478,400 | Train Loss EWMA: 6.2601 | Learning Rate: 0.002000 | Progress: 0.82739
+Step 5,075 | Tokens: 166,297,600 | Train Loss EWMA: 6.2630 | Learning Rate: 0.002000 | Progress: 0.83149
+Step 5,100 | Tokens: 167,116,800 | Train Loss EWMA: 6.2739 | Learning Rate: 0.002000 | Progress: 0.83558
+Step 5,125 | Tokens: 167,936,000 | Train Loss EWMA: 6.2439 | Learning Rate: 0.002000 | Progress: 0.83968
+Step 5,150 | Tokens: 168,755,200 | Train Loss EWMA: 6.2397 | Learning Rate: 0.002000 | Progress: 0.84378
+Step 5,175 | Tokens: 169,574,400 | Train Loss EWMA: 6.2190 | Learning Rate: 0.002000 | Progress: 0.84787
+Step 5,200 | Tokens: 170,393,600 | Train Loss EWMA: 6.2233 | Learning Rate: 0.002000 | Progress: 0.85197
+Step 5,225 | Tokens: 171,212,800 | Train Loss EWMA: 6.2213 | Learning Rate: 0.002000 | Progress: 0.85606
+Step 5,250 | Tokens: 172,032,000 | Train Loss EWMA: 6.2204 | Learning Rate: 0.002000 | Progress: 0.86016
+Step 5,275 | Tokens: 172,851,200 | Train Loss EWMA: 6.2177 | Learning Rate: 0.002000 | Progress: 0.86426
+Step 5,300 | Tokens: 173,670,400 | Train Loss EWMA: 6.2225 | Learning Rate: 0.002000 | Progress: 0.86835
+Step 5,325 | Tokens: 174,489,600 | Train Loss EWMA: 6.2290 | Learning Rate: 0.002000 | Progress: 0.87245
+Step 5,350 | Tokens: 175,308,800 | Train Loss EWMA: 6.2401 | Learning Rate: 0.002000 | Progress: 0.87654
+Step 5,375 | Tokens: 176,128,000 | Train Loss EWMA: 6.2484 | Learning Rate: 0.002000 | Progress: 0.88064
+Step 5,400 | Tokens: 176,947,200 | Train Loss EWMA: 6.2459 | Learning Rate: 0.002000 | Progress: 0.88474
+Step 5,425 | Tokens: 177,766,400 | Train Loss EWMA: 6.2543 | Learning Rate: 0.002000 | Progress: 0.88883
+Step 5,450 | Tokens: 178,585,600 | Train Loss EWMA: 6.2470 | Learning Rate: 0.002000 | Progress: 0.89293
+Step 5,475 | Tokens: 179,404,800 | Train Loss EWMA: 6.2508 | Learning Rate: 0.002000 | Progress: 0.89702
+Step 5,500 | Tokens: 180,224,000 | Train Loss EWMA: 6.2458 | Learning Rate: 0.002000 | Progress: 0.90112
+Step 5,525 | Tokens: 181,043,200 | Train Loss EWMA: 6.2487 | Learning Rate: 0.002000 | Progress: 0.90522
+Step 5,550 | Tokens: 181,862,400 | Train Loss EWMA: 6.2461 | Learning Rate: 0.002000 | Progress: 0.90931
+Step 5,575 | Tokens: 182,681,600 | Train Loss EWMA: 6.2378 | Learning Rate: 0.002000 | Progress: 0.91341
+Step 5,600 | Tokens: 183,500,800 | Train Loss EWMA: 6.2363 | Learning Rate: 0.002000 | Progress: 0.91750
+Step 5,625 | Tokens: 184,320,000 | Train Loss EWMA: 6.2206 | Learning Rate: 0.002000 | Progress: 0.92160
+Step 5,650 | Tokens: 185,139,200 | Train Loss EWMA: 6.2282 | Learning Rate: 0.002000 | Progress: 0.92570
+Step 5,675 | Tokens: 185,958,400 | Train Loss EWMA: 6.2391 | Learning Rate: 0.002000 | Progress: 0.92979
+Step 5,700 | Tokens: 186,777,600 | Train Loss EWMA: 6.2379 | Learning Rate: 0.002000 | Progress: 0.93389
+Step 5,725 | Tokens: 187,596,800 | Train Loss EWMA: 6.2373 | Learning Rate: 0.002000 | Progress: 0.93798
+Step 5,750 | Tokens: 188,416,000 | Train Loss EWMA: 6.2270 | Learning Rate: 0.002000 | Progress: 0.94208
+Step 5,775 | Tokens: 189,235,200 | Train Loss EWMA: 6.2179 | Learning Rate: 0.002000 | Progress: 0.94618
+Step 5,800 | Tokens: 190,054,400 | Train Loss EWMA: 6.2114 | Learning Rate: 0.002000 | Progress: 0.95027
+Step 5,825 | Tokens: 190,873,600 | Train Loss EWMA: 6.2080 | Learning Rate: 0.002000 | Progress: 0.95437
+Step 5,850 | Tokens: 191,692,800 | Train Loss EWMA: 6.2047 | Learning Rate: 0.002000 | Progress: 0.95846
+Step 5,875 | Tokens: 192,512,000 | Train Loss EWMA: 6.2303 | Learning Rate: 0.002000 | Progress: 0.96256
+Step 5,900 | Tokens: 193,331,200 | Train Loss EWMA: 6.2244 | Learning Rate: 0.002000 | Progress: 0.96666
+Step 5,925 | Tokens: 194,150,400 | Train Loss EWMA: 6.2344 | Learning Rate: 0.002000 | Progress: 0.97075
+Step 5,950 | Tokens: 194,969,600 | Train Loss EWMA: 6.2312 | Learning Rate: 0.002000 | Progress: 0.97485
+Step 5,975 | Tokens: 195,788,800 | Train Loss EWMA: 6.2234 | Learning Rate: 0.002000 | Progress: 0.97894
+Step 6,000 | Tokens: 196,608,000 | Train Loss EWMA: 6.2024 | Learning Rate: 0.002000 | Progress: 0.98304
+Step 6,025 | Tokens: 197,427,200 | Train Loss EWMA: 6.1917 | Learning Rate: 0.002000 | Progress: 0.98714
+Step 6,050 | Tokens: 198,246,400 | Train Loss EWMA: 6.1895 | Learning Rate: 0.002000 | Progress: 0.99123
+Step 6,075 | Tokens: 199,065,600 | Train Loss EWMA: 6.1859 | Learning Rate: 0.002000 | Progress: 0.99533
+Step 6,100 | Tokens: 199,884,800 | Train Loss EWMA: 6.1768 | Learning Rate: 0.002000 | Progress: 0.99942
diff --git a/wandb/run-20250926_233554-wxnxan9r/files/requirements.txt b/wandb/run-20250926_233554-wxnxan9r/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ba4ea9984d2e87ab2846009f43584e075c3b3e7b
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/files/requirements.txt
@@ -0,0 +1,215 @@
+fsspec==2025.3.0
+PyYAML==6.0.2
+certifi==2025.8.3
+comm==0.2.3
+widgetsnbextension==4.0.14
+Jinja2==3.1.6
+rich==14.1.0
+circuitsvis==1.43.3
+hf-xet==1.1.9
+param==2.2.1
+httpcore==1.0.9
+nvidia-cuda-cupti-cu12==12.8.90
+nvidia-cuda-nvrtc-cu12==12.8.93
+asttokens==3.0.0
+filelock==3.19.1
+types-python-dateutil==2.9.0.20250822
+cycler==0.12.1
+stack-data==0.6.3
+jupyter_server==2.17.0
+aiosignal==1.4.0
+xyzservices==2025.4.0
+lark==1.2.2
+ptyprocess==0.7.0
+xxhash==3.5.0
+mpmath==1.3.0
+seaborn==0.13.2
+wadler_lindig==0.1.7
+nbformat==5.10.4
+panel==1.8.0
+accelerate==1.10.1
+plotly==6.3.0
+narwhals==2.4.0
+huggingface-hub==0.34.4
+sentencepiece==0.2.1
+torchvision==0.23.0
+ipython==9.5.0
+tqdm==4.67.1
+contourpy==1.3.3
+nvidia-nvtx-cu12==12.8.90
+nvidia-cuda-runtime-cu12==12.8.90
+yarl==1.20.1
+charset-normalizer==3.4.3
+jupyter-events==0.12.0
+nbclient==0.10.2
+numpy==1.26.4
+decorator==5.2.1
+networkx==3.5
+smmap==5.0.2
+nbconvert==7.16.6
+pytz==2025.2
+aiohappyeyeballs==2.6.1
+requests==2.32.5
+tinycss2==1.4.0
+defusedxml==0.7.1
+matplotlib-inline==0.1.7
+rpds-py==0.27.1
+wandb==0.21.4
+jedi==0.19.2
+pathspec==0.12.1
+transformer-lens==2.16.1
+sympy==1.14.0
+jupyterlab_pygments==0.3.0
+overrides==7.7.0
+notebook_shim==0.2.4
+jupyter==1.1.1
+protobuf==6.32.1
+better-abc==0.0.3
+jsonpointer==3.0.0
+terminado==0.18.1
+cfgv==3.4.0
+rfc3987-syntax==1.1.0
+annotated-types==0.7.0
+pyarrow==21.0.0
+webencodings==0.5.1
+wcwidth==0.2.13
+jupyterlab_server==2.27.3
+argon2-cffi-bindings==25.1.0
+nvidia-nvjitlink-cu12==12.8.93
+jaxtyping==0.3.2
+Pygments==2.19.2
+torch==2.8.0
+rfc3339-validator==0.1.4
+urllib3==2.5.0
+jupyterlab_widgets==3.0.15
+ipykernel==6.30.1
+nvidia-cudnn-cu12==9.10.2.21
+beautifulsoup4==4.13.5
+babel==2.17.0
+pure_eval==0.2.3
+pyparsing==3.2.3
+nvidia-cublas-cu12==12.8.4.1
+regex==2025.9.1
+pycparser==2.23
+soupsieve==2.8
+pytest-cov==7.0.0
+sniffio==1.3.1
+mypy==1.18.1
+notebook==7.4.5
+packaging==25.0
+h11==0.16.0
+psutil==7.0.0
+pexpect==4.9.0
+gitdb==4.0.12
+rfc3986-validator==0.1.1
+pyzmq==27.1.0
+jupyterlab==4.4.7
+toy_models==0.1.0
+torchaudio==2.8.0
+cffi==2.0.0
+mypy_extensions==1.1.0
+attrs==25.3.0
+transformers==4.56.1
+jupyter_core==5.8.1
+bleach==6.2.0
+fqdn==1.5.1
+async-lru==2.0.5
+nvidia-nccl-cu12==2.27.3
+GitPython==3.1.45
+referencing==0.36.2
+click==8.2.1
+prometheus_client==0.22.1
+bokeh==3.8.0
+httpx==0.28.1
+setuptools==80.9.0
+argon2-cffi==25.1.0
+multidict==6.6.4
+pyviz_comms==3.0.6
+arrow==1.3.0
+beartype==0.14.1
+ipywidgets==8.1.7
+pydantic_core==2.33.2
+markdown-it-py==4.0.0
+pandas==2.3.2
+virtualenv==20.34.0
+python-dotenv==1.1.1
+isoduration==20.11.0
+python-dateutil==2.9.0.post0
+nodeenv==1.9.1
+nvidia-curand-cu12==10.3.9.90
+webcolors==24.11.1
+MarkupSafe==3.0.2
+nvidia-cusolver-cu12==11.7.3.90
+Send2Trash==1.8.3
+coverage==7.10.6
+jupyter_server_terminals==0.5.3
+debugpy==1.8.16
+json5==0.12.1
+linkify-it-py==2.0.3
+importlib_metadata==8.7.0
+nvidia-cufft-cu12==11.3.3.83
+distlib==0.4.0
+typing-inspection==0.4.1
+identify==2.6.14
+nvidia-cufile-cu12==1.13.1.3
+mdurl==0.1.2
+websocket-client==1.8.0
+jsonschema==4.25.1
+python-json-logger==3.3.0
+typing_extensions==4.15.0
+tokenizers==0.22.0
+ipympl==0.9.7
+einops==0.8.1
+jupyter_client==8.6.3
+ipython_pygments_lexers==1.1.1
+h5py==3.14.0
+tabulate==0.9.0
+propcache==0.3.2
+ruff==0.13.0
+tornado==6.5.2
+typeguard==4.4.4
+tomlkit==0.13.2
+pluggy==1.6.0
+pydantic==2.11.7
+zipp==3.23.0
+fancy-einsum==0.0.3
+fastjsonschema==2.21.2
+datasets==4.0.0
+fonttools==4.59.2
+executing==2.2.1
+pillow==11.3.0
+uc-micro-py==1.0.3
+Markdown==3.9
+pre_commit==4.3.0
+aiohttp==3.12.15
+mistune==3.1.4
+tzdata==2025.2
+parso==0.8.5
+triton==3.4.0
+kiwisolver==1.4.9
+idna==3.10
+multiprocess==0.70.16
+dill==0.3.8
+jupyter-lsp==2.3.0
+platformdirs==4.4.0
+sentry-sdk==2.37.1
+prompt_toolkit==3.0.52
+jsonschema-specifications==2025.9.1
+pytest==8.4.2
+mdit-py-plugins==0.5.0
+transformers-stream-generator==0.0.5
+nvidia-cusparselt-cu12==0.7.1
+pandocfilters==1.5.1
+jupyter-console==6.6.3
+anyio==4.10.0
+six==1.17.0
+holoviews==1.21.0
+matplotlib==3.10.6
+colorcet==3.1.0
+uri-template==1.3.0
+nest-asyncio==1.6.0
+nvidia-cusparse-cu12==12.5.8.93
+iniconfig==2.1.0
+traitlets==5.14.3
+safetensors==0.6.2
+frozenlist==1.7.0
diff --git a/wandb/run-20250926_233554-wxnxan9r/files/wandb-metadata.json b/wandb/run-20250926_233554-wxnxan9r/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..e645426b473f2e1fc813f207a7881de7f41d0cea
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/files/wandb-metadata.json
@@ -0,0 +1,38 @@
+{
+  "os":  "Linux-5.19.0-45-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.7",
+  "startedAt":  "2025-09-26T23:35:54.126498Z",
+  "program":  "<python with no main file>",
+  "git":  {
+    "remote":  "https://github.com/jgroh3/toy_models.git",
+    "commit":  "750b6c98784324c15d9d813c9e365e5ccfa8b0ab"
+  },
+  "email":  "tzfof8@gmail.com",
+  "root":  "/notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2",
+  "host":  "nq6cv1dtxr",
+  "executable":  "/notebooks/toy_models/.toy_models_env/bin/python",
+  "cpu_count":  8,
+  "cpu_count_logical":  8,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "262240792576",
+      "used":  "154771664896"
+    }
+  },
+  "memory":  {
+    "total":  "47332843520"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-7b3bcd99-03fe-0e7c-86cb-d143c0cb514f"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "s2y1kfnn063xv2a8fa1iwgztn27t3afe"
+}
\ No newline at end of file
diff --git a/wandb/run-20250926_233554-wxnxan9r/files/wandb-summary.json b/wandb/run-20250926_233554-wxnxan9r/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..b9c2516c8cd191b239e0391c4018c95e82b457ed
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/files/wandb-summary.json
@@ -0,0 +1 @@
+{"tokens_per_second":32768,"_step":6100,"train_loss_ewma":6.176772246384147,"train_loss":6.255190849304199,"step":6100,"_runtime":4094.066844279,"progress":0.999424,"_timestamp":1.7589338455948539e+09,"tokens_seen":199884800,"learning_rate":0.002,"_wandb":{"runtime":4094}}
\ No newline at end of file
diff --git a/wandb/run-20250926_233554-wxnxan9r/logs/debug-core.log b/wandb/run-20250926_233554-wxnxan9r/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..5b5aad091a690470922958864af38630d26fa192
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/logs/debug-core.log
@@ -0,0 +1,16 @@
+{"time":"2025-09-26T23:35:54.385394045Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp6bv7f6l5/port-540.txt","pid":540,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-26T23:35:54.386037992Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":540}
+{"time":"2025-09-26T23:35:54.386006382Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-540-603-2322522254/socket","Net":"unix"}}
+{"time":"2025-09-26T23:35:54.387687307Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-26T23:35:54.403117295Z","level":"INFO","msg":"handleInformInit: received","streamId":"wxnxan9r","id":"1(@)"}
+{"time":"2025-09-26T23:35:54.609812237Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"wxnxan9r","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.296615424Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"wxnxan9r","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.299416387Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"wxnxan9r","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.299441662Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.29945742Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.299470957Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-27T00:44:09.299472928Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.299514306Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.299519601Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-27T00:44:09.299576812Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-540-603-2322522254/socket","Net":"unix"}}
+{"time":"2025-09-27T00:44:09.29959761Z","level":"INFO","msg":"server is closed"}
diff --git a/wandb/run-20250926_233554-wxnxan9r/logs/debug-internal.log b/wandb/run-20250926_233554-wxnxan9r/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c1c6beec2e82bd8bb1b3f38d57d48ece39d02d05
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/logs/debug-internal.log
@@ -0,0 +1,12 @@
+{"time":"2025-09-26T23:35:54.403287329Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-26T23:35:54.609764327Z","level":"INFO","msg":"stream: created new stream","id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609806816Z","level":"INFO","msg":"stream: started","id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609823798Z","level":"INFO","msg":"writer: started","stream_id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609841971Z","level":"INFO","msg":"handler: started","stream_id":"wxnxan9r"}
+{"time":"2025-09-26T23:35:54.609869143Z","level":"INFO","msg":"sender: started","stream_id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:08.838815981Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":0.022747613}],"total_operations":1}}
+{"time":"2025-09-27T00:44:09.172329967Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-27T00:44:09.297375375Z","level":"INFO","msg":"stream: closing","id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:09.297412196Z","level":"INFO","msg":"handler: closed","stream_id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:09.297486349Z","level":"INFO","msg":"sender: closed","stream_id":"wxnxan9r"}
+{"time":"2025-09-27T00:44:09.297492707Z","level":"INFO","msg":"stream: closed","id":"wxnxan9r"}
diff --git a/wandb/run-20250926_233554-wxnxan9r/logs/debug.log b/wandb/run-20250926_233554-wxnxan9r/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..4676117587174aa747f6212dba2c6ddca74aa7fa
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/logs/debug.log
@@ -0,0 +1,26 @@
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Configure stats pid to 540
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Loading settings from /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2/wandb/settings
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:setup_run_log_directory():686] Logging user logs to /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2/wandb/run-20250926_233554-wxnxan9r/logs/debug.log
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to /notebooks/toy_models/model_training/c4_SeqUnmodified_NonSeqObserved_L2/wandb/run-20250926_233554-wxnxan9r/logs/debug-internal.log
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:init():813] calling init triggers
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'c4_SeqUnmodified_NonSeqObserved_L2', 'n_layers': 2, 'd_model': 512, 'd_mlp': 2048, 'd_head': 64, 'n_heads': 8, 'attn_only': False, 'layer_norm_eps': 1e-05, 'init_range': 0.02, 'n_ctx': 1024, 'd_vocab': 48262, 'dataset_name': 'eoinf/c4_SeqUnmodified_NonSeqObserved_L2', 'tokenizer_name': 'NeelNanda/gpt-neox-tokenizer-digits', 'seed': 10, 'device': 'cuda', 'use_bfloat16_matmul': False, 'batch_size_per_device': 32, 'n_devices': 1, 'batches_per_step': 1, 'max_tokens': 200000000, 'lr_hidden': 0.002, 'lr_vector': 0.001, 'lr_schedule': 'constant_with_warmup', 'warmup_tokens': 30000000, 'weight_decay': 0.05, 'grad_norm_clip': 1.0, 'train_loss_moving_average_beta': 0.99, 'log_interval': 25, 'save_checkpoints': True, 'checkpoint_interval': 500, 'checkpoint_interval_ratio': 1.1, 'save_log_checkpoints': True, 'use_wandb': True, 'batch_size': 32, 'tokens_per_step': 32768, 'warmup_steps': 915, 'max_steps': 6103, '_wandb': {}}
+2025-09-26 23:35:54,130 INFO    MainThread:540 [wandb_init.py:init():854] starting backend
+2025-09-26 23:35:54,387 INFO    MainThread:540 [wandb_init.py:init():857] sending inform_init request
+2025-09-26 23:35:54,398 INFO    MainThread:540 [wandb_init.py:init():865] backend started and connected
+2025-09-26 23:35:54,399 INFO    MainThread:540 [wandb_init.py:init():936] updated telemetry
+2025-09-26 23:35:54,439 INFO    MainThread:540 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-26 23:35:54,734 INFO    MainThread:540 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_console_start():2506] atexit reg
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2025-09-26 23:35:55,244 INFO    MainThread:540 [wandb_run.py:_redirect():2446] Redirects installed.
+2025-09-26 23:35:55,252 INFO    MainThread:540 [wandb_init.py:init():1049] run started, returning control to user process
+2025-09-27 00:44:08,798 INFO    MainThread:540 [wandb_run.py:_finish():2272] finishing run tzach/toy-transformer-replication/wxnxan9r
+2025-09-27 00:44:08,800 INFO    MainThread:540 [wandb_run.py:_atexit_cleanup():2471] got exitcode: 0
+2025-09-27 00:44:08,800 INFO    MainThread:540 [wandb_run.py:_restore():2453] restore
+2025-09-27 00:44:08,800 INFO    MainThread:540 [wandb_run.py:_restore():2459] restore done
+2025-09-27 00:44:09,296 INFO    MainThread:540 [wandb_run.py:_footer_sync_info():3867] logging synced files
diff --git a/wandb/run-20250926_233554-wxnxan9r/run-wxnxan9r.wandb b/wandb/run-20250926_233554-wxnxan9r/run-wxnxan9r.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..96c7bbd4cf5c38868fc84a1f1b8640d46d0555b5
--- /dev/null
+++ b/wandb/run-20250926_233554-wxnxan9r/run-wxnxan9r.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aecad41419f097c60600c948233b8e4a9307f1d4046e777ac36b3ce4c7026e3
+size 4776503