diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/run.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/run.json
new file mode 100644
index 0000000000000000000000000000000000000000..22cae601fff997614b79d9d643aeedffadae8411
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/run.json
@@ -0,0 +1,10 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "4f01f653a56d0daf",
+  "wandb_run_id": "95708491",
+  "created_at": 1783965749
+}
diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/summary.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..33528ff8d4ed4de47d56d2a6fe01345060890dcd
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/summary.json
@@ -0,0 +1,93 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "dataset": "jbduran/think-dataset-clean-1930s",
+  "dataset_revision": "main",
+  "step": 2362,
+  "depth": 12,
+  "target_param_data_ratio": 11.25,
+  "training_tokens": 1238368256,
+  "final_sampled_val_bpb": 1.1162452479490097,
+  "minimum_sampled_val_bpb": 1.1162452479490097,
+  "full_val_bpb": 1.0583351355174524,
+  "core_metric": 0.06305335515020447,
+  "centered_results": {
+    "hellaswag_zeroshot": 0.036181370417277016,
+    "jeopardy": 0.0004723665479104966,
+    "bigbench_qa_wikidata": 0.08601938933134079,
+    "arc_easy": 0.0953984260559082,
+    "arc_challenge": -0.04550625880559286,
+    "copa": 0.0,
+    "commonsense_qa": 0.10421784967184065,
+    "piqa": 0.08487486839294434,
+    "openbook_qa": -0.02399998903274536,
+    "lambada_openai": 0.21191538870334625,
+    "hellaswag": 0.03910243511199951,
+    "winograd": 0.040293097496032715,
+    "winogrande": 0.0007892847061157227,
+    "bigbench_dyck_languages": 0.07200000435113907,
+    "agi_eval_lsat_ar": 0.07608695328235625,
+    "bigbench_cs_algorithms": 0.4371212124824524,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.020813623443245888,
+    "coqa": 0.07303018867969513,
+    "boolq": -0.2071463622544941,
+    "bigbench_language_identification": 0.18074805875553682
+  },
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the kingdom of France, and the capital of the kingdom of England"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the gold of the earth, and the gold of the earth is the gold of"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day will be Sunday. The day will be Sunday, and"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the same as that of cold, and the same as that of hot. The"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the color of the sky, and the color of the earth. The color of"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the sum of the sum of the sum of the sum of the sum of the"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>CHAPTER VS. CREV. A. It has already been mentioned that Roger North and the Anglo- Saxon Chronicle and the reigns of Henry and Edward III. tell us that Eden was situated on England's Apple within the government of Richard II., and it must now be premised that England brought over men from Normandy and Anjou, and maintained them there. The folly and swiftness of our pilgrims were its source.\n\nProvidence was no idler in being driven over this island by the rivers of Canterbury and Canterbury, and thence overland to Rome; nor in being conveyed to Jerusalem, to meet the friendlessness of an ignorant man",
+    "<|bos|>PREFACE.\n\nTHE portions of this work, which, without modification of time, would be impossible to be within the little compass of the compass, are so valuable, that it must be judged more as a class; and all men, of every class, are judged by the same standards of skill, observed by the same standard of practice.\n\nThe first part of the Cloaca Maxima, which was written in print, is too exuberant and exuberant, to allow of annotation. \"But the question came up whilst we were considering the books,\" he adds, \"of Ministers, And the gentlemen that were about me, joined in the",
+    "<|bos|>PREFACE.\n\nix emphatic word is given to such relief; and in the language of filial affection or maternal emotion or affection, the language he employs is never \"the language of all true repentance and deep sorrow, but a resort\" of sighs or complaints to a Father to whom all are alike reconciled, whose infinite attributes are majesty, loving-kindness, mercy, and justice.\n\nIf the incipient man was a material object or an object, he would \"stalk;\" by indolence, by vice, by attraction, he would throw himself into a stagnant pool, throw into it little basins and merely onions, drenching",
+    "<|bos|> Dreams and Rejoice over my Lute and Pledge I've Soon Began to\n\nWish never the pains of keep as Oblation all my Steps I took to Narrow the bounds I had to set;\n\nExtanced at this Stage my Branding spowl yet\n\nSeemed it a rather\n\nSpeech and gn\n\nA considerable Quantity Flapping in my 80 whiteness forbid the Onely False ity did afford me;\n\nAnd whoever Fidelity's Man bring in all South about 100\n\nEtched upon Conscience an Evoluble Sport Gold upon there a Tremb Billail he Built I had little faith till",
+    "<|bos|>Drops, with apples nourishing them and lilies suiting them. It is a custom with corn-growers to mow loaves. It is thus with the rusted corn, which we call wheat because it is hard. If a may be properly branched out into layers and used in raising and other purposes, it is sometimes described as branched out simply for bed, and germinated like a cap or knife, into one or other of the ultimate layers. It sometimes happens that pipes are left, which being laid on the ground and decayed or damped in time as they decompose them, remain",
+    "<|bos|>VE ZABE. [Heir of the Superinto- FOW is a gener- Founding of the Morality | Sociessie, by the Origin of Lan- Turks Previous to his elevation strength of Language, very spe- ties, been endowed with this essential business, ago there are only three distinct untitutional faculties. But the reconstruction of modern sciences in the Shinto variant is a very daring one, and treats the sublimity of the Fourteenth century as such. Storia in three skins parapped off byila- Strumeton, Nev., drawn him away and merrie (Wat v. Ortsboth",
+    "<|bos|>PREFACE Ruth and Robin Hood suggested the latter an asylum in the outhouses at Oakum Orbeek, where the late Charles Lee, the publisher of 1 A little instalment of this and other historical works is printed at the end of 1872.\n\nPREFACE.\n\nto secure a place for retirement should he be ever called on to join the rebels. This was left the further task of keeping all the patent offices in the selection of Warwick the publisher of aegicide Liber Quintale. \"Master Pashan and the vae variarent,\" added Francis Gorton after the interview above to Pashan's title,",
+    "<|bos|>opers.\n\n CarthABELLA LUBI.\n\n221 however, still perseveres (1251 n. d.), and succeeded in breaking down the walls and overwhelming the rest with stones. 66 It appears,\" says Lubio (d.) I, \"that the free city is secure in its centre; and that I, who have interfered on behalf of my people at all times during my rule, can easily turn the safety of Smyrna when made famous by the massacre of the Amir and the surrender of the city on the appointed day.\" This is somewhat like about Cortez: \"Franciscus qui desitunt\n\n"
+  ],
+  "training_time_seconds": 6341.746661424637,
+  "stage_training_flops": 1.0985538644638433e+18,
+  "inherited_parent_flops": 0.0,
+  "cumulative_pipeline_training_flops": 1.0985538644638433e+18,
+  "config_fingerprint": "4f01f653a56d0daf",
+  "git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/95708491",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-muoneq-muonplus",
+  "dataset_fingerprint": "4af5759948d15ff0",
+  "tokenizer_fingerprint": "803338edbf3458e6",
+  "unique_train_tokens": 1275519304,
+  "effective_epochs": 0.970873786164196
+}
diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/experiment_tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..4552b41249154dd89202bb69f5cb6e659f2e2b46
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/experiment_tokenizer.json
@@ -0,0 +1,18 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 26,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "created_at": 1783965799
+}
diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/token_bytes.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26d7e6581c41a049b038de82b8bb65a1c8d57c8f
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/token_bytes.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d25a126690db557828b34dc3bd3e8ebd615058016c1870e76b32b31266805ed
+size 132649
diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/tokenizer.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..d21e36e06c64fe9e86d8d579bccaae5542ad5ac9
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/tokenizer.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4932125f789267dbd571cc4bd19339ecfd7160c9b9cc0441ba6635756e4a8b0e
+size 407747
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_000500.json
new file mode 100644
index 0000000000000000000000000000000000000000..334540825140b855909c94e40ec689b724d138fb
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_000500.json
@@ -0,0 +1,142 @@
+{
+  "step": 500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "val_bpb": 1.3561541455317183,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25-randtok",
+    "wandb_run_id": "6b8f19a5",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25-randtok",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
+    "tokenizer_fingerprint": "f01549dbe6e2aaa9",
+    "git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25-randtok",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25-randtok",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768,
+        "sampling": "random",
+        "sampling_seed": 42
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25-randtok",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25",
+          "randtok"
+        ]
+      },
+      "config_fingerprint": "175184ba2ee0f5d3",
+      "artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "175184ba2ee0f5d3"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.3561541455317183,
+    "smooth_train_loss": 3.8651697814606103,
+    "total_training_time": 1305.9748463630676,
+    "stage_training_flops": 232547388751872000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 232547388751872000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001000.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7929f9639e59948cc65075c39652c8f12f07d
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001000.json
@@ -0,0 +1,142 @@
+{
+  "step": 1000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "val_bpb": 1.2521959168139378,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25-randtok",
+    "wandb_run_id": "6b8f19a5",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25-randtok",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
+    "tokenizer_fingerprint": "f01549dbe6e2aaa9",
+    "git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25-randtok",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25-randtok",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768,
+        "sampling": "random",
+        "sampling_seed": 42
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25-randtok",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25",
+          "randtok"
+        ]
+      },
+      "config_fingerprint": "175184ba2ee0f5d3",
+      "artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "175184ba2ee0f5d3"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2521959168139378,
+    "smooth_train_loss": 3.5352641959529243,
+    "total_training_time": 2640.971613883972,
+    "stage_training_flops": 465094777503744000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 465094777503744000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001500.json
new file mode 100644
index 0000000000000000000000000000000000000000..f28c0f7122e1dd22a855fdf7f364beb013489ea3
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001500.json
@@ -0,0 +1,142 @@
+{
+  "step": 1500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "val_bpb": 1.171838291812204,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25-randtok",
+    "wandb_run_id": "6b8f19a5",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25-randtok",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
+    "tokenizer_fingerprint": "f01549dbe6e2aaa9",
+    "git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25-randtok",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25-randtok",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768,
+        "sampling": "random",
+        "sampling_seed": 42
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25-randtok",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25",
+          "randtok"
+        ]
+      },
+      "config_fingerprint": "175184ba2ee0f5d3",
+      "artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "175184ba2ee0f5d3"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86488769,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86488769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.171838291812204,
+    "smooth_train_loss": 3.4159228083544537,
+    "total_training_time": 3976.167183637619,
+    "stage_training_flops": 697642166255616000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 697642166255616000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002000.json
new file mode 100644
index 0000000000000000000000000000000000000000..22dd0864f21492fdbd26dc3d1feb46718eb406e2
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002000.json
@@ -0,0 +1,142 @@
+{
+  "step": 2000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "val_bpb": 1.1368966310332098,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25-randtok",
+    "wandb_run_id": "6b8f19a5",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25-randtok",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
+    "tokenizer_fingerprint": "f01549dbe6e2aaa9",
+    "git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25-randtok",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25-randtok",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768,
+        "sampling": "random",
+        "sampling_seed": 42
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25-randtok",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25",
+          "randtok"
+        ]
+      },
+      "config_fingerprint": "175184ba2ee0f5d3",
+      "artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "175184ba2ee0f5d3"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48640769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48640769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1368966310332098,
+    "smooth_train_loss": 3.136166264429775,
+    "total_training_time": 5310.99071097374,
+    "stage_training_flops": 930189555007488000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 930189555007488000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002362.json
new file mode 100644
index 0000000000000000000000000000000000000000..e761b66bdcd3a934d3e44f0e27cf21122551cd1c
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002362.json
@@ -0,0 +1,142 @@
+{
+  "step": 2362,
+  "training_complete": true,
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "val_bpb": 1.1128382305031206,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25-randtok",
+    "wandb_run_id": "6b8f19a5",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25-randtok",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
+    "tokenizer_fingerprint": "f01549dbe6e2aaa9",
+    "git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25-randtok",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25-randtok",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768,
+        "sampling": "random",
+        "sampling_seed": 42
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25-randtok",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25",
+          "randtok"
+        ]
+      },
+      "config_fingerprint": "175184ba2ee0f5d3",
+      "artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "175184ba2ee0f5d3"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 12,
+    "pos": 38438817,
+    "epoch": 1,
+    "pq_idx": 12,
+    "rg_idx": 38438817
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1128382305031206,
+    "smooth_train_loss": 3.185300296294561,
+    "total_training_time": 6277.177423000336,
+    "stage_training_flops": 1098553864463843328,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1098553864463843328
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4216bdba9af7eb6db9d6b4efc4564092bba4c7eb
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae8a604031071534c2cd16d6086cd495012f38121b53e2072da33e084a8b3ba0
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c48611ab7e81a8a1766e50d63591462824e3f46
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48630f071b47f0d89ea2572c606d32e32b1c143d345176add9aeaf81483ea490
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..def9e1c80987e41395a2c1ecf96d24ff359c6263
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1359363c9b03f7dd9ae84fb5a606e06dfaaaeabf612168817daa388780ca4ed4
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3654468cd5c86915b8e6ef23ed4952a82885b34a
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cd9a38e9ecae268247a5a41d1710293cf7c9176b74660d0d3a36a86a9770302
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002362.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d1c6296224fb9f2bc3140f7f2b0e606c6c59682
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002362.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a72dbad509f7969cc0c8825a3078a402e31a466476b39573b8c638ac3d37a519
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_000500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3fa0fb3e5d38a530d918d37936d031ddd23b8b0c
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_000500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:443ccb4c4d76adf1c33e0d0b3cacad3f29f522f07643ee180e6f461178593f51
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..629925ba1bffcd2113f7930bd69e2dbacc714085
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36325a3bba2b4d7461ae926bf79112585c2e52c5bf1c9f5847f5198713794add
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57722c59bc97301d345a708ca7f6a3110e061358
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bb11ab9d0b1d54523b190dea5f37893ebd91858694403d047606196aa16887d
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e362508889ac81fc9fbf6c3f26b122e5b339602a
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6941a6e8678fc86b1041e974eed1173410cca4fea06167c76e96a64d5260f6a
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002362_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fde42ad8707f958b368d1ffaedacbe1dc0e8c36b
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002362_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0648d159d91067bff35675cba8b5828e792b5ef7eb442e6ec5ea883179e65a12
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25-randtok/config.json b/experiments/clean1930s-d12-r11.25-randtok/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..a830707f7c7496c04dc047a0fcf2c99a4d2fe841
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/config.json
@@ -0,0 +1,59 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 26,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768,
+    "sampling": "random",
+    "sampling_seed": 42
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 12,
+    "scaling_params": 110100912,
+    "target_param_data_ratio": 11.25,
+    "window_pattern": "L",
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "seed": 42,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano"
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "clean1930s-d12-r11.25-randtok",
+    "group": "clean1930s-d12",
+    "tags": [
+      "think-dataset-clean-1930s",
+      "d12",
+      "ratio11.25",
+      "randtok"
+    ]
+  },
+  "config_fingerprint": "175184ba2ee0f5d3",
+  "artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
+}
diff --git a/experiments/clean1930s-d12-r11.25-randtok/evals/core.json b/experiments/clean1930s-d12-r11.25-randtok/evals/core.json
new file mode 100644
index 0000000000000000000000000000000000000000..48f06d21a6672fd3fe140f024b8b13e99fdeba94
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/evals/core.json
@@ -0,0 +1,56 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": 0.05120713426476866,
+  "core_results": {
+    "hellaswag_zeroshot": 0.2790280878543854,
+    "jeopardy": 0.0004723665479104966,
+    "bigbench_qa_wikidata": 0.07145317643880844,
+    "arc_easy": 0.31607744097709656,
+    "arc_challenge": 0.20563139021396637,
+    "copa": 0.4899999797344208,
+    "commonsense_qa": 0.3054873049259186,
+    "piqa": 0.5429815053939819,
+    "openbook_qa": 0.23200000822544098,
+    "lambada_openai": 0.19153890013694763,
+    "hellaswag": 0.27853015065193176,
+    "winograd": 0.5604395866394043,
+    "winogrande": 0.49486976861953735,
+    "bigbench_dyck_languages": 0.08400000631809235,
+    "agi_eval_lsat_ar": 0.30869564414024353,
+    "bigbench_cs_algorithms": 0.4204545319080353,
+    "bigbench_operators": 0.07619047909975052,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.021948911249637604,
+    "coqa": 0.05273706465959549,
+    "boolq": 0.430886834859848,
+    "bigbench_language_identification": 0.2540999948978424
+  },
+  "centered_results": {
+    "hellaswag_zeroshot": 0.038704117139180504,
+    "jeopardy": 0.0004723665479104966,
+    "bigbench_qa_wikidata": 0.07145317643880844,
+    "arc_easy": 0.08810325463612874,
+    "arc_challenge": -0.059158146381378174,
+    "copa": -0.020000040531158447,
+    "commonsense_qa": 0.1318591311573982,
+    "piqa": 0.08596301078796387,
+    "openbook_qa": -0.02399998903274536,
+    "lambada_openai": 0.19153890013694763,
+    "hellaswag": 0.03804020086924235,
+    "winograd": 0.1208791732788086,
+    "winogrande": -0.010260462760925293,
+    "bigbench_dyck_languages": 0.08400000631809235,
+    "agi_eval_lsat_ar": 0.13586955517530439,
+    "bigbench_cs_algorithms": 0.4204545319080353,
+    "bigbench_operators": 0.07619047909975052,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.021948911249637604,
+    "coqa": 0.05273706465959549,
+    "boolq": -0.4976662240530315,
+    "bigbench_language_identification": 0.17942793718134478
+  },
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/evals/samples.json b/experiments/clean1930s-d12-r11.25-randtok/evals/samples.json
new file mode 100644
index 0000000000000000000000000000000000000000..30566f62be7b55eea29b81cefb1504b54aa520c9
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/evals/samples.json
@@ -0,0 +1,48 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the nation, and the capital of the nation is the capital of"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the same as that of silver, and the latter is the same as the former"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day of the week will be Sunday. The day of the"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the most common form of the word. The word is used in the sense of"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is a very fine one, and I have never seen it in my life. I"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the parts of the body, and x is the number of the"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>ccliary on a certain phase of life-into which the crews of vessels of war have their chief and are fortunate in holding the chief place at the point of the stroke-what second seeks life artillery fortification in the future, the young shoot; and resisting a longer bullet-wounded tale in order to conform to the demands of business, declare openly, what is the duty of these young officers. 339\n\nINTRODUCTION\n\nAn indefatigable fellow who has attained middle life experiences abundant satisfaction no less keen on such occasions than when a fighter's son with ardour in the fray will, having crowned his freshness with laurels of strong individuality and shouting the",
+    "<|bos|>CHAPTER XVIII.\n\nPersonal Voyages in Shropshire.- Voyage to Lourdes.-Inland Voyages to Madison County. - Impressments upon the northern New\n\nHampshire Army.-The King entertained Chancellor Kent.-The French Mail-Scope.\n\n-Comparison of Narrative given by Prince Hoofloy. - Encounters and villages of the Church of Lourdes.-No Severance of Early Days.-The\n\nEnglish secure coal from Pilgrim Hall.--Mr. De Chaumier's\"-Public Windows.- Cholera of the Country.-(Robert Hall's)-Return to London before Alvare",
+    "<|bos|>PREFACE.\n\nix it in the seventh book to the lines full and true. ritie Mc-\n\nMullen, receiving theory kinde of his essay on \"The Heart and the Heart.\"\n\nSomebody has written of my scholarship, and mine ability. But no apology cately based on facts has beamed from the lips of Shobaugh: 'Tis ake win' and 'tis ake ake win\n\nBy early discovery eke of the common 'tall Hhor.'\n\nYoung Horace, who stands as fellow-musical with the immigration of\n\nClarkes and Iredellians; was born in the Pacific Alps",
+    "<|bos|>The company will draw on its rent at Five Pence.\n\nTail & Doors set up for Assoc. J. Glasgow Modern Greek\n\nSons, Parts 16 and 17.\n\nTECHNICAL PICTURES.\n\nIntroductory.\n\nThe writer of the foregoing points out two systems of pillaging as the result: one for intent and one for specialty, regardless of cost and charges; but as elements in analysis, the aims of the permisi Ph. and 3 years\n\nTail & Dooors loud. $ength. 61 a. 50 asses emmen. 25 phied ",
+    "<|bos|>ENDORION COURTISEMENT\n\nCOPYRIGHT, 1901, BY\n\nPHILIP S. ROGERS Esquire to the Honorable the Honorable the Board of Admiralty, Admiralty, &c.,:\n\n(MARY\n\nN. ROGERS Mary Heptam. The Hon. H. H.\n\nMackintosh.\n\nThe Honorable the Honorable the Honorable the Honorable the Queen Margot, and the Countess of\n\nPoultney, do certify that the following are the names of the persons named in the above-mentioned proposals, and the quantities of gold the said persons have Ibrahim the Harriethbout King etc., which",
+    "<|bos|>onica, 1. 3. 22 and weissen, 233, 4 Supr. 455.\n\nYAWARD, See BARPAIN ON.\n\n1 On my official report of my public duties in chief, Nays Lincoln, Boston, To-day, 31st December, 1861. [28th June, 1862] in\n\nWAR DEPARTMENT,\n\nWASHINGTON, June 10, 1863. \"MY DEAR BROTHER: By letter I received yesterday the letter written to you, father, father, and uncle, and sent you because you do not belong to them who wrote",
+    "<|bos|>AV-Government of the Six Nations, so long as it exists, is materially weak, and may be ultimately superseded, by the Government of Modern India itself; and a ministry in England may turn it in its administration into a formidable force, which it cannot cope with in India, as a subordinate agency, by extraordinary administrative talents, required for its focus; and in default may render futile any further offensive or nugatory effort to the natives of India. As a temporary agent, the Council put itself under the protection of\n\nJacob Grimm, an Edinburgh man, who, like him, was under orders.\n\nThe senior partner of the same",
+    "<|bos|>PREFACE.\n\nI scruple not to reiterate this impression, and to acquiesce, with grateful emotions, in the misfortunes which late, my dear Mr. Ballantyne, have Providence permitted for my own sake.\n\nIt is impossible that the author of a set of printed memoirs relating to Scotland could not find an excuse for having discovered his friend's address on a paper called undoubtedly \"The Popular Musick Services,\" published in the year-book of the year 1822 --a paper wholly in blank, for sale in Kensington Gardens, where it happened to have that particular merit, and on whose reception he thought it only right to use his name whenever"
+  ]
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-randtok/evals/val_bpb.json
new file mode 100644
index 0000000000000000000000000000000000000000..66632cb7c068f4ceb610c63811c0921e572f2bf8
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/evals/val_bpb.json
@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val": 1.053600922794229
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25-randtok/run.json b/experiments/clean1930s-d12-r11.25-randtok/run.json
new file mode 100644
index 0000000000000000000000000000000000000000..7df4a3f4999cff938d74c156bedd87dddd4d16bd
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/run.json
@@ -0,0 +1,10 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "175184ba2ee0f5d3",
+  "wandb_run_id": "6b8f19a5",
+  "created_at": 1784310529
+}
diff --git a/experiments/clean1930s-d12-r11.25-randtok/summary.json b/experiments/clean1930s-d12-r11.25-randtok/summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..0acee228c7f834705004a4bc0ea6cdf2c9a624bf
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/summary.json
@@ -0,0 +1,93 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r11.25-randtok",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "dataset": "jbduran/think-dataset-clean-1930s",
+  "dataset_revision": "main",
+  "step": 2362,
+  "depth": 12,
+  "target_param_data_ratio": 11.25,
+  "training_tokens": 1238368256,
+  "final_sampled_val_bpb": 1.1128382305031206,
+  "minimum_sampled_val_bpb": 1.1128382305031206,
+  "full_val_bpb": 1.053600922794229,
+  "core_metric": 0.05120713426476866,
+  "centered_results": {
+    "hellaswag_zeroshot": 0.038704117139180504,
+    "jeopardy": 0.0004723665479104966,
+    "bigbench_qa_wikidata": 0.07145317643880844,
+    "arc_easy": 0.08810325463612874,
+    "arc_challenge": -0.059158146381378174,
+    "copa": -0.020000040531158447,
+    "commonsense_qa": 0.1318591311573982,
+    "piqa": 0.08596301078796387,
+    "openbook_qa": -0.02399998903274536,
+    "lambada_openai": 0.19153890013694763,
+    "hellaswag": 0.03804020086924235,
+    "winograd": 0.1208791732788086,
+    "winogrande": -0.010260462760925293,
+    "bigbench_dyck_languages": 0.08400000631809235,
+    "agi_eval_lsat_ar": 0.13586955517530439,
+    "bigbench_cs_algorithms": 0.4204545319080353,
+    "bigbench_operators": 0.07619047909975052,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.021948911249637604,
+    "coqa": 0.05273706465959549,
+    "boolq": -0.4976662240530315,
+    "bigbench_language_identification": 0.17942793718134478
+  },
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the nation, and the capital of the nation is the capital of"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the same as that of silver, and the latter is the same as the former"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day of the week will be Sunday. The day of the"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the most common form of the word. The word is used in the sense of"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is a very fine one, and I have never seen it in my life. I"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the parts of the body, and x is the number of the"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>ccliary on a certain phase of life-into which the crews of vessels of war have their chief and are fortunate in holding the chief place at the point of the stroke-what second seeks life artillery fortification in the future, the young shoot; and resisting a longer bullet-wounded tale in order to conform to the demands of business, declare openly, what is the duty of these young officers. 339\n\nINTRODUCTION\n\nAn indefatigable fellow who has attained middle life experiences abundant satisfaction no less keen on such occasions than when a fighter's son with ardour in the fray will, having crowned his freshness with laurels of strong individuality and shouting the",
+    "<|bos|>CHAPTER XVIII.\n\nPersonal Voyages in Shropshire.- Voyage to Lourdes.-Inland Voyages to Madison County. - Impressments upon the northern New\n\nHampshire Army.-The King entertained Chancellor Kent.-The French Mail-Scope.\n\n-Comparison of Narrative given by Prince Hoofloy. - Encounters and villages of the Church of Lourdes.-No Severance of Early Days.-The\n\nEnglish secure coal from Pilgrim Hall.--Mr. De Chaumier's\"-Public Windows.- Cholera of the Country.-(Robert Hall's)-Return to London before Alvare",
+    "<|bos|>PREFACE.\n\nix it in the seventh book to the lines full and true. ritie Mc-\n\nMullen, receiving theory kinde of his essay on \"The Heart and the Heart.\"\n\nSomebody has written of my scholarship, and mine ability. But no apology cately based on facts has beamed from the lips of Shobaugh: 'Tis ake win' and 'tis ake ake win\n\nBy early discovery eke of the common 'tall Hhor.'\n\nYoung Horace, who stands as fellow-musical with the immigration of\n\nClarkes and Iredellians; was born in the Pacific Alps",
+    "<|bos|>The company will draw on its rent at Five Pence.\n\nTail & Doors set up for Assoc. J. Glasgow Modern Greek\n\nSons, Parts 16 and 17.\n\nTECHNICAL PICTURES.\n\nIntroductory.\n\nThe writer of the foregoing points out two systems of pillaging as the result: one for intent and one for specialty, regardless of cost and charges; but as elements in analysis, the aims of the permisi Ph. and 3 years\n\nTail & Dooors loud. $ength. 61 a. 50 asses emmen. 25 phied ",
+    "<|bos|>ENDORION COURTISEMENT\n\nCOPYRIGHT, 1901, BY\n\nPHILIP S. ROGERS Esquire to the Honorable the Honorable the Board of Admiralty, Admiralty, &c.,:\n\n(MARY\n\nN. ROGERS Mary Heptam. The Hon. H. H.\n\nMackintosh.\n\nThe Honorable the Honorable the Honorable the Honorable the Queen Margot, and the Countess of\n\nPoultney, do certify that the following are the names of the persons named in the above-mentioned proposals, and the quantities of gold the said persons have Ibrahim the Harriethbout King etc., which",
+    "<|bos|>onica, 1. 3. 22 and weissen, 233, 4 Supr. 455.\n\nYAWARD, See BARPAIN ON.\n\n1 On my official report of my public duties in chief, Nays Lincoln, Boston, To-day, 31st December, 1861. [28th June, 1862] in\n\nWAR DEPARTMENT,\n\nWASHINGTON, June 10, 1863. \"MY DEAR BROTHER: By letter I received yesterday the letter written to you, father, father, and uncle, and sent you because you do not belong to them who wrote",
+    "<|bos|>AV-Government of the Six Nations, so long as it exists, is materially weak, and may be ultimately superseded, by the Government of Modern India itself; and a ministry in England may turn it in its administration into a formidable force, which it cannot cope with in India, as a subordinate agency, by extraordinary administrative talents, required for its focus; and in default may render futile any further offensive or nugatory effort to the natives of India. As a temporary agent, the Council put itself under the protection of\n\nJacob Grimm, an Edinburgh man, who, like him, was under orders.\n\nThe senior partner of the same",
+    "<|bos|>PREFACE.\n\nI scruple not to reiterate this impression, and to acquiesce, with grateful emotions, in the misfortunes which late, my dear Mr. Ballantyne, have Providence permitted for my own sake.\n\nIt is impossible that the author of a set of printed memoirs relating to Scotland could not find an excuse for having discovered his friend's address on a paper called undoubtedly \"The Popular Musick Services,\" published in the year-book of the year 1822 --a paper wholly in blank, for sale in Kensington Gardens, where it happened to have that particular merit, and on whose reception he thought it only right to use his name whenever"
+  ],
+  "training_time_seconds": 6277.177423000336,
+  "stage_training_flops": 1.0985538644638433e+18,
+  "inherited_parent_flops": 0.0,
+  "cumulative_pipeline_training_flops": 1.0985538644638433e+18,
+  "config_fingerprint": "175184ba2ee0f5d3",
+  "git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/6b8f19a5",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-randtok",
+  "dataset_fingerprint": "4af5759948d15ff0",
+  "tokenizer_fingerprint": "f01549dbe6e2aaa9",
+  "unique_train_tokens": 1275519304,
+  "effective_epochs": 0.970873786164196
+}
diff --git a/experiments/clean1930s-d12-r11.25-randtok/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-randtok/tokenizer/experiment_tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..0de3544da519fa2a4ac3bd71b929f5ae5a3f417e
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/tokenizer/experiment_tokenizer.json
@@ -0,0 +1,20 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25-randtok",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 26,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768,
+    "sampling": "random",
+    "sampling_seed": 42
+  },
+  "created_at": 1784310546
+}
diff --git a/experiments/clean1930s-d12-r11.25-randtok/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-randtok/tokenizer/token_bytes.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea66127b8045ccb5e30dc43f57f9812d849460fb
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/tokenizer/token_bytes.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9de5ef70a8805ae1e23ed607d05ab79cf92582f123294be962ca714ff0664ec
+size 132649
diff --git a/experiments/clean1930s-d12-r11.25-randtok/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-randtok/tokenizer/tokenizer.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..c0c31da99f03bdb9a149b516c2d48c10ec57a9bd
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25-randtok/tokenizer/tokenizer.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21d846dc85121d501a10b42cacd77297a6358e43203f03c320f64c2c395ba894
+size 411592
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_000500.json
new file mode 100644
index 0000000000000000000000000000000000000000..e3f4ea070fbd4660e2685b3b42239d46bdb0a0dc
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_000500.json
@@ -0,0 +1,139 @@
+{
+  "step": 500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25",
+  "val_bpb": 1.3406276768636045,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25",
+    "wandb_run_id": "764ab08e",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
+    "tokenizer_fingerprint": "fa7b147cb0e4b5db",
+    "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25"
+        ]
+      },
+      "config_fingerprint": "00e7ae6b9109c167",
+      "artifact_path": "experiments/clean1930s-d12-r11.25"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "00e7ae6b9109c167"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.3406276768636045,
+    "smooth_train_loss": 3.8475896185653875,
+    "total_training_time": 1308.1742749214172,
+    "stage_training_flops": 232547388751872000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 232547388751872000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001000.json
new file mode 100644
index 0000000000000000000000000000000000000000..a49512a2120e8c2614992870f1c4f2afb7ff88c6
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001000.json
@@ -0,0 +1,139 @@
+{
+  "step": 1000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25",
+  "val_bpb": 1.2453749523249924,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25",
+    "wandb_run_id": "764ab08e",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
+    "tokenizer_fingerprint": "fa7b147cb0e4b5db",
+    "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25"
+        ]
+      },
+      "config_fingerprint": "00e7ae6b9109c167",
+      "artifact_path": "experiments/clean1930s-d12-r11.25"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "00e7ae6b9109c167"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2453749523249924,
+    "smooth_train_loss": 3.5828240195598844,
+    "total_training_time": 2642.554317712784,
+    "stage_training_flops": 465094777503744000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 465094777503744000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001500.json
new file mode 100644
index 0000000000000000000000000000000000000000..96dd3dc4993bc92fb7d27680fae439ce2723eb0e
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001500.json
@@ -0,0 +1,139 @@
+{
+  "step": 1500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25",
+  "val_bpb": 1.1784689301618998,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25",
+    "wandb_run_id": "764ab08e",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
+    "tokenizer_fingerprint": "fa7b147cb0e4b5db",
+    "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25"
+        ]
+      },
+      "config_fingerprint": "00e7ae6b9109c167",
+      "artifact_path": "experiments/clean1930s-d12-r11.25"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "00e7ae6b9109c167"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86488769,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86488769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1784689301618998,
+    "smooth_train_loss": 3.3444808841561646,
+    "total_training_time": 3977.2887001037598,
+    "stage_training_flops": 697642166255616000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 697642166255616000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002000.json
new file mode 100644
index 0000000000000000000000000000000000000000..4263a4e47ff3cf48cd0cbd573f064cb92c9d9362
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002000.json
@@ -0,0 +1,139 @@
+{
+  "step": 2000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r11.25",
+  "val_bpb": 1.1385790516794956,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25",
+    "wandb_run_id": "764ab08e",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
+    "tokenizer_fingerprint": "fa7b147cb0e4b5db",
+    "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25"
+        ]
+      },
+      "config_fingerprint": "00e7ae6b9109c167",
+      "artifact_path": "experiments/clean1930s-d12-r11.25"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "00e7ae6b9109c167"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48640769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48640769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1385790516794956,
+    "smooth_train_loss": 3.19688287657793,
+    "total_training_time": 5312.674092292786,
+    "stage_training_flops": 930189555007488000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 930189555007488000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002362.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c41191c4d687aa13902f6b143e76f406d04c2f1
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002362.json
@@ -0,0 +1,139 @@
+{
+  "step": 2362,
+  "training_complete": true,
+  "experiment_id": "clean1930s-d12-r11.25",
+  "val_bpb": 1.1175529448105965,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r11.25",
+    "wandb_run_id": "764ab08e",
+    "wandb_group": "clean1930s-d12",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r11.25",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
+    "tokenizer_fingerprint": "fa7b147cb0e4b5db",
+    "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r11.25",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r11.25",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 26,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r11.25",
+        "group": "clean1930s-d12",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio11.25"
+        ]
+      },
+      "config_fingerprint": "00e7ae6b9109c167",
+      "artifact_path": "experiments/clean1930s-d12-r11.25"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r11.25",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "00e7ae6b9109c167"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 12,
+    "pos": 38438817,
+    "epoch": 1,
+    "pq_idx": 12,
+    "rg_idx": 38438817
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1175529448105965,
+    "smooth_train_loss": 3.1313785378510484,
+    "total_training_time": 6279.889491558075,
+    "stage_training_flops": 1098553864463843328,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1098553864463843328
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ceb2b4169ca2a7d5e8a0d9777522a5b96a0693d
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e818606e95665b2d93328ebac939eff1721dfddd5bc633c670d05644ec02b09
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93685362662a8925999a9a1c22e31c3e7391e50a
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6afac069374a34152cb01eff7142a0e285d70880bee793b10c0de073805a3864
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..012c06c2b1386831b274f6e87ddf44c548057d06
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1da723ae288e3aedd7142082491758bd1a6be3791cb4b881f32909c63e22ec68
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b479dd3d324ef8af976ec600f87afe56ada089e
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:108865e4d197f8c5f893fc9e0ef7568bf33258fa908fd843002e4033436e6169
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_002362.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e349db1b9a56456c799c2d3ec26bc6805cd6edff
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/model_002362.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66478807ab543dc52a021b0989d39b1f59fdb9978bc679dc23781c97b0136a23
+size 792761690
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_000500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2195a9fc6b51e60c259798dd16e2305120faf409
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_000500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a57bc43f0fc66aa44a67432373a76db9ee4be1b9b425ec037327188641df180d
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ddf177d9fce73af3f8558903ccd6c4aad0fe481b
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5876a82a2083edec62f32471e1e8cd87749d59d9663905c7762447601e480bb7
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c8169d0f893f94927b95381eea9124869b18e62f
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46edf74481ffc6508a1abdf1ad0837b2b1d1fefb6d09328fed8067b7819df19a
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42c474cb7a1c58bb6019fa157a31e55eff6188a6
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c13b9d6a8e454e20f62c5f1ea69c9076a2baffe739b3af61569b029c3203fe33
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002362_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d53108e4c979b5a82cb8068289d4e4b66c3c274
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002362_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a09823e14e88be9d6acaea7b9d3a2e5c88b435d1d40bbcbe656f780c874cdb32
+size 1246165357
diff --git a/experiments/clean1930s-d12-r11.25/config.json b/experiments/clean1930s-d12-r11.25/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..579f1648757616da1f6cde7bce72504ad8d6b386
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/config.json
@@ -0,0 +1,56 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "clean1930s-d12-r11.25",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 26,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 12,
+    "scaling_params": 110100912,
+    "target_param_data_ratio": 11.25,
+    "window_pattern": "L",
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "seed": 42,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano"
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "clean1930s-d12-r11.25",
+    "group": "clean1930s-d12",
+    "tags": [
+      "think-dataset-clean-1930s",
+      "d12",
+      "ratio11.25"
+    ]
+  },
+  "config_fingerprint": "00e7ae6b9109c167",
+  "artifact_path": "experiments/clean1930s-d12-r11.25"
+}
diff --git a/experiments/clean1930s-d12-r11.25/evals/core.json b/experiments/clean1930s-d12-r11.25/evals/core.json
new file mode 100644
index 0000000000000000000000000000000000000000..264ffc5cda224fa6fdf951ac0fd176d7cdf8429d
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/evals/core.json
@@ -0,0 +1,56 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": 0.06238580602029931,
+  "core_results": {
+    "hellaswag_zeroshot": 0.2756423056125641,
+    "jeopardy": 0.0,
+    "bigbench_qa_wikidata": 0.07450420409440994,
+    "arc_easy": 0.32281145453453064,
+    "arc_challenge": 0.21587030589580536,
+    "copa": 0.5399999618530273,
+    "commonsense_qa": 0.26535627245903015,
+    "piqa": 0.5402611494064331,
+    "openbook_qa": 0.242000013589859,
+    "lambada_openai": 0.2115272581577301,
+    "hellaswag": 0.2757418751716614,
+    "winograd": 0.5641025900840759,
+    "winogrande": 0.49723756313323975,
+    "bigbench_dyck_languages": 0.1120000034570694,
+    "agi_eval_lsat_ar": 0.2913043200969696,
+    "bigbench_cs_algorithms": 0.41969695687294006,
+    "bigbench_operators": 0.06190476566553116,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.014758750796318054,
+    "coqa": 0.05574345588684082,
+    "boolq": 0.4880733788013458,
+    "bigbench_language_identification": 0.2556000053882599
+  },
+  "centered_results": {
+    "hellaswag_zeroshot": 0.034189740816752114,
+    "jeopardy": 0.0,
+    "bigbench_qa_wikidata": 0.07450420409440994,
+    "arc_easy": 0.09708193937937419,
+    "arc_challenge": -0.04550625880559286,
+    "copa": 0.07999992370605469,
+    "commonsense_qa": 0.08169534057378768,
+    "piqa": 0.08052229881286621,
+    "openbook_qa": -0.010666648546854654,
+    "lambada_openai": 0.2115272581577301,
+    "hellaswag": 0.034322500228881836,
+    "winograd": 0.12820518016815186,
+    "winogrande": -0.005524873733520508,
+    "bigbench_dyck_languages": 0.1120000034570694,
+    "agi_eval_lsat_ar": 0.11413040012121199,
+    "bigbench_cs_algorithms": 0.41969695687294006,
+    "bigbench_operators": 0.06190476566553116,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.014758750796318054,
+    "coqa": 0.05574345588684082,
+    "boolq": -0.34717531894382675,
+    "bigbench_language_identification": 0.18107811373845972
+  },
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/evals/samples.json b/experiments/clean1930s-d12-r11.25/evals/samples.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7289a8e77a728a06536f0b3a4b89a7913385f1a
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/evals/samples.json
@@ -0,0 +1,48 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the Empire, and the capital of the Empire is the capital of"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the \"milk of the earth,\" and the \"milk of"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday was Saturday, then tomorrow will be Saturday. If yesterday was"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the hot, and the hot is the cold. The hot is the hot,"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: 1. The sun, 2. The moon, 3. The"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the red of the red of the red of the red of the red of the"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of days in which the earth is in motion, and the number of"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>CHAPTER V FORMS ON STEAM MILLS, It has already been pointed out that the very good Anglo- Saxon capital and its effects are of recent introduction in the East. The consciousness of modernity in architecture is rapidly \"tasking.\" In Europe and America, no single innovation, no single change in the least tenuous architecture, would be tolerated. On the contrary, so many hardy pioneers of civilization now lie dead in the mud or are idly rotting in stones; so general an application would still be to a period prior to 1870, just as we now to a century since, are inclined to reckon an ignorant man",
+    "<|bos|>PREFACE.\n\nso often quoted and described, and revealed the fact of average, almost unknowable, within the little knowable field of opinion of his.\n\nLet me hope that we may go as far as the narrative will admit into the explanation of such expressions as The Genealogies of Defendant, The Book-hunter, The Wise Man, etc.\n\nIt is often said, and so I deem it, that the gentleman who draws the following is not moving on his account, and does not design the question.\n\nLet us say, to the reader, This pantheistic text furnishes a full and complete explanation of the about the differences which make",
+    "<|bos|>PREFACE.\n\nix emphatic word is my wish. with one or two reasonable exceptions, and who experiences any emotion or fear on hearing of a narrative of her \"Franks.\"\n\nA strong and authoritative book, on a vouchsafed subject, or novelty or tedium to human beliefes careles of this life of yours shall be the common wish of every man, and it will make the incipient man that child alone that compilers, with the fore-power of vision, have by indolence, or vice, or other powerful means, precipitated the history ofori- this society.\"MI- MENTS FROM THE JUDGmeign. I",
+    "<|bos|> Dreams and Reverses, by Rev. P. I. Sheard.\n\n(ACT IV.) WALL RUN RECREATIONS\n\nIII Last Summer Evening, I took a Narrow Way to Princeton, Indiana, the night after Shamokoka fired their last shot, in their settlement and near Spring Hill, near\n\nAkosta Flats. Throughout the day whitening the trees around, tinged the spires of farms seated on clumps of trees and in all directions about them, and we had a fine time. Evidently to begin with, there came a large party of our men out of the lines, among",
+    "<|bos|>Dieta, i. 653, 686, 692.\n\nCook's ear, i. 76-80, 167, 235, 236.\n\nDoor-watchage, i. 82, 83, 113.\n\nDoolittle's reading, i. 548.\n\nDoomer's simply court notice, i. 355.\n\nEhambra parting teeth, i. 700, 702.. Ehambra weeping, i. 146. e.\n\nEast Bangalore, ii. 374.\n\nEcclesiastics",
+    "<|bos|>orms, while the moral principle is allowable in ordinary life. The boys belonging to a tradesman's home are let alone after spending a few weeks, during which he does not receive any pay; this exhausts their labors. Since there are no other means for carrying through business, few will be willing to undertake it; but few display the ability that can sustain sustained excitement by the prompt bestowal of a generous payment.\n\nThe third day is occupied in operaucb Reading. Sticks have been hired for over six months for delivery at a drug store, but this payment has been withheld because otherwise (Wat v. Ortsville",
+    "<|bos|>) Ruth and Robin lick'll look an' put up o' Tal-ledge's our followlin' lar, Iween, a 're luther Tol a little afraid 'll stay 'ead a-dandy at a P'ock harness an' es 'to Awlfe Cl Proper should slavery me sent, Weritan To! Cane flax can Gither it ble at Cutnish Maalock the times of aeething, Some lady misses to be able to Pane the vaeuce ground, some lady rod theows hast' a to Pate downt wi the",
+    "<|bos|> shares ought to be limited by the terms of a deed and however they may be in the hands of numerous individuals, the sum can easily be made large by even a few executed persons alone. Let the number of such persons gathered and the results produced justify the conclusion that the shares were free from all liabilities in their hands, and that their share had a legal amount sufficient to cover them at all times.\n\nThe rule adopted by the American Accountant General for Improving the Powers of the President of the United States Government in regard to the use of legal instruments for the transaction of business in the execution of contracts, applies as well to such legal"
+  ]
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25/evals/val_bpb.json
new file mode 100644
index 0000000000000000000000000000000000000000..36642bfe6a681b55ed411f548a6c880b2e611c22
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/evals/val_bpb.json
@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val": 1.0596072239127994
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset-clean.json b/experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset-clean.json
new file mode 100644
index 0000000000000000000000000000000000000000..4111f11a1e12fe1407f26ba775d0fab4d9087f93
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset-clean.json
@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val": 1.06282022394172
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset.json b/experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba1f719b5724a5f39190b16170e12252c6519481
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset.json
@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val": 1.1058237779626336
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r11.25/run.json b/experiments/clean1930s-d12-r11.25/run.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b6360678536c9e2ae96a6b1160a4595bcf3534a
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/run.json
@@ -0,0 +1,10 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r11.25",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "00e7ae6b9109c167",
+  "wandb_run_id": "764ab08e",
+  "created_at": 1783954179
+}
diff --git a/experiments/clean1930s-d12-r11.25/summary.json b/experiments/clean1930s-d12-r11.25/summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd1692d698923853bfe8658980af0643d1cb650a
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/summary.json
@@ -0,0 +1,93 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r11.25",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "dataset": "jbduran/think-dataset-clean-1930s",
+  "dataset_revision": "main",
+  "step": 2362,
+  "depth": 12,
+  "target_param_data_ratio": 11.25,
+  "training_tokens": 1238368256,
+  "final_sampled_val_bpb": 1.1175529448105965,
+  "minimum_sampled_val_bpb": 1.1175529448105965,
+  "full_val_bpb": 1.0596072239127994,
+  "core_metric": 0.06238580602029931,
+  "centered_results": {
+    "hellaswag_zeroshot": 0.034189740816752114,
+    "jeopardy": 0.0,
+    "bigbench_qa_wikidata": 0.07450420409440994,
+    "arc_easy": 0.09708193937937419,
+    "arc_challenge": -0.04550625880559286,
+    "copa": 0.07999992370605469,
+    "commonsense_qa": 0.08169534057378768,
+    "piqa": 0.08052229881286621,
+    "openbook_qa": -0.010666648546854654,
+    "lambada_openai": 0.2115272581577301,
+    "hellaswag": 0.034322500228881836,
+    "winograd": 0.12820518016815186,
+    "winogrande": -0.005524873733520508,
+    "bigbench_dyck_languages": 0.1120000034570694,
+    "agi_eval_lsat_ar": 0.11413040012121199,
+    "bigbench_cs_algorithms": 0.41969695687294006,
+    "bigbench_operators": 0.06190476566553116,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.014758750796318054,
+    "coqa": 0.05574345588684082,
+    "boolq": -0.34717531894382675,
+    "bigbench_language_identification": 0.18107811373845972
+  },
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the Empire, and the capital of the Empire is the capital of"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the \"milk of the earth,\" and the \"milk of"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday was Saturday, then tomorrow will be Saturday. If yesterday was"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the hot, and the hot is the cold. The hot is the hot,"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: 1. The sun, 2. The moon, 3. The"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the red of the red of the red of the red of the red of the"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of days in which the earth is in motion, and the number of"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>CHAPTER V FORMS ON STEAM MILLS, It has already been pointed out that the very good Anglo- Saxon capital and its effects are of recent introduction in the East. The consciousness of modernity in architecture is rapidly \"tasking.\" In Europe and America, no single innovation, no single change in the least tenuous architecture, would be tolerated. On the contrary, so many hardy pioneers of civilization now lie dead in the mud or are idly rotting in stones; so general an application would still be to a period prior to 1870, just as we now to a century since, are inclined to reckon an ignorant man",
+    "<|bos|>PREFACE.\n\nso often quoted and described, and revealed the fact of average, almost unknowable, within the little knowable field of opinion of his.\n\nLet me hope that we may go as far as the narrative will admit into the explanation of such expressions as The Genealogies of Defendant, The Book-hunter, The Wise Man, etc.\n\nIt is often said, and so I deem it, that the gentleman who draws the following is not moving on his account, and does not design the question.\n\nLet us say, to the reader, This pantheistic text furnishes a full and complete explanation of the about the differences which make",
+    "<|bos|>PREFACE.\n\nix emphatic word is my wish. with one or two reasonable exceptions, and who experiences any emotion or fear on hearing of a narrative of her \"Franks.\"\n\nA strong and authoritative book, on a vouchsafed subject, or novelty or tedium to human beliefes careles of this life of yours shall be the common wish of every man, and it will make the incipient man that child alone that compilers, with the fore-power of vision, have by indolence, or vice, or other powerful means, precipitated the history ofori- this society.\"MI- MENTS FROM THE JUDGmeign. I",
+    "<|bos|> Dreams and Reverses, by Rev. P. I. Sheard.\n\n(ACT IV.) WALL RUN RECREATIONS\n\nIII Last Summer Evening, I took a Narrow Way to Princeton, Indiana, the night after Shamokoka fired their last shot, in their settlement and near Spring Hill, near\n\nAkosta Flats. Throughout the day whitening the trees around, tinged the spires of farms seated on clumps of trees and in all directions about them, and we had a fine time. Evidently to begin with, there came a large party of our men out of the lines, among",
+    "<|bos|>Dieta, i. 653, 686, 692.\n\nCook's ear, i. 76-80, 167, 235, 236.\n\nDoor-watchage, i. 82, 83, 113.\n\nDoolittle's reading, i. 548.\n\nDoomer's simply court notice, i. 355.\n\nEhambra parting teeth, i. 700, 702.. Ehambra weeping, i. 146. e.\n\nEast Bangalore, ii. 374.\n\nEcclesiastics",
+    "<|bos|>orms, while the moral principle is allowable in ordinary life. The boys belonging to a tradesman's home are let alone after spending a few weeks, during which he does not receive any pay; this exhausts their labors. Since there are no other means for carrying through business, few will be willing to undertake it; but few display the ability that can sustain sustained excitement by the prompt bestowal of a generous payment.\n\nThe third day is occupied in operaucb Reading. Sticks have been hired for over six months for delivery at a drug store, but this payment has been withheld because otherwise (Wat v. Ortsville",
+    "<|bos|>) Ruth and Robin lick'll look an' put up o' Tal-ledge's our followlin' lar, Iween, a 're luther Tol a little afraid 'll stay 'ead a-dandy at a P'ock harness an' es 'to Awlfe Cl Proper should slavery me sent, Weritan To! Cane flax can Gither it ble at Cutnish Maalock the times of aeething, Some lady misses to be able to Pane the vaeuce ground, some lady rod theows hast' a to Pate downt wi the",
+    "<|bos|> shares ought to be limited by the terms of a deed and however they may be in the hands of numerous individuals, the sum can easily be made large by even a few executed persons alone. Let the number of such persons gathered and the results produced justify the conclusion that the shares were free from all liabilities in their hands, and that their share had a legal amount sufficient to cover them at all times.\n\nThe rule adopted by the American Accountant General for Improving the Powers of the President of the United States Government in regard to the use of legal instruments for the transaction of business in the execution of contracts, applies as well to such legal"
+  ],
+  "training_time_seconds": 6279.889491558075,
+  "stage_training_flops": 1.0985538644638433e+18,
+  "inherited_parent_flops": 0.0,
+  "cumulative_pipeline_training_flops": 1.0985538644638433e+18,
+  "config_fingerprint": "00e7ae6b9109c167",
+  "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/764ab08e",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25",
+  "dataset_fingerprint": "4af5759948d15ff0",
+  "tokenizer_fingerprint": "fa7b147cb0e4b5db",
+  "unique_train_tokens": 1275519304,
+  "effective_epochs": 0.970873786164196
+}
diff --git a/experiments/clean1930s-d12-r11.25/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25/tokenizer/experiment_tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f01bf637a5554656b2e99480af66a9565d17ef7
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/tokenizer/experiment_tokenizer.json
@@ -0,0 +1,18 @@
+{
+  "experiment_id": "clean1930s-d12-r11.25",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 26,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "created_at": 1783954241
+}
diff --git a/experiments/clean1930s-d12-r11.25/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25/tokenizer/token_bytes.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26d7e6581c41a049b038de82b8bb65a1c8d57c8f
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/tokenizer/token_bytes.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d25a126690db557828b34dc3bd3e8ebd615058016c1870e76b32b31266805ed
+size 132649
diff --git a/experiments/clean1930s-d12-r11.25/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25/tokenizer/tokenizer.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..d21e36e06c64fe9e86d8d579bccaae5542ad5ac9
--- /dev/null
+++ b/experiments/clean1930s-d12-r11.25/tokenizer/tokenizer.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4932125f789267dbd571cc4bd19339ecfd7160c9b9cc0441ba6635756e4a8b0e
+size 407747
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_000500.json
new file mode 100644
index 0000000000000000000000000000000000000000..7fdbf177e459ceb0f896d2f649bc42a2ce23b414
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_000500.json
@@ -0,0 +1,222 @@
+{
+  "step": 500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "val_bpb": 1.3394909610264625,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r12-ctx4096",
+    "wandb_run_id": "e564d552",
+    "wandb_group": "clean1930s-d12-ctx4096",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": 2520,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok",
+    "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r12-ctx4096",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "23031c7d2f7cc08da131639554d952aef02069e9",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r12-ctx4096",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r12-ctx4096",
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 180,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 78,
+          "num_train_shards": 40,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 101,
+          "num_train_shards": 20,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 1321205760,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 924844032,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 1189085184,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 524288,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r12-ctx4096",
+        "group": "clean1930s-d12-ctx4096",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio12",
+          "ctx4096",
+          "midtrain-schedule",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "f54a41c1b7b43ef4",
+      "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "f54a41c1b7b43ef4",
+    "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769,
+    "mixture": {
+      "cursors": {
+        "original": {
+          "file_idx": 2,
+          "pos": 62184769,
+          "epoch": 1,
+          "pq_idx": 2,
+          "rg_idx": 62184769
+        },
+        "midtrain_r30": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 262176768,
+      "source_tokens": {
+        "original": 262176768,
+        "midtrain_r30": 0,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 0
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 1.3394909610264625,
+    "smooth_train_loss": 3.7008363848340475,
+    "total_training_time": 1494.5915577411652,
+    "stage_training_flops": 291921016651776000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 291921016651776000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_001000.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb85a501067ae9a9bbfaef5feb445ac054b9c377
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_001000.json
@@ -0,0 +1,222 @@
+{
+  "step": 1000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "val_bpb": 1.2353599635607029,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r12-ctx4096",
+    "wandb_run_id": "e564d552",
+    "wandb_group": "clean1930s-d12-ctx4096",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": 2520,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok",
+    "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r12-ctx4096",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "23031c7d2f7cc08da131639554d952aef02069e9",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r12-ctx4096",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r12-ctx4096",
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 180,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 78,
+          "num_train_shards": 40,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 101,
+          "num_train_shards": 20,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 1321205760,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 924844032,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 1189085184,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 524288,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r12-ctx4096",
+        "group": "clean1930s-d12-ctx4096",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio12",
+          "ctx4096",
+          "midtrain-schedule",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "f54a41c1b7b43ef4",
+      "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "f54a41c1b7b43ef4",
+    "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769,
+    "mixture": {
+      "cursors": {
+        "original": {
+          "file_idx": 5,
+          "pos": 24336769,
+          "epoch": 1,
+          "pq_idx": 5,
+          "rg_idx": 24336769
+        },
+        "midtrain_r30": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 524320768,
+      "source_tokens": {
+        "original": 524320768,
+        "midtrain_r30": 0,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 0
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2353599635607029,
+    "smooth_train_loss": 3.5236556054731762,
+    "total_training_time": 3021.539851665497,
+    "stage_training_flops": 583842033303552000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 583842033303552000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_001500.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb77d29275aeb07f42cbaeaa74478e07b4716356
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_001500.json
@@ -0,0 +1,222 @@
+{
+  "step": 1500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "val_bpb": 1.1534062642313452,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r12-ctx4096",
+    "wandb_run_id": "e564d552",
+    "wandb_group": "clean1930s-d12-ctx4096",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": 2520,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": 1000,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok",
+    "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r12-ctx4096",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "23031c7d2f7cc08da131639554d952aef02069e9",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r12-ctx4096",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r12-ctx4096",
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 180,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 78,
+          "num_train_shards": 40,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 101,
+          "num_train_shards": 20,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 1321205760,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 924844032,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 1189085184,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 524288,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r12-ctx4096",
+        "group": "clean1930s-d12-ctx4096",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio12",
+          "ctx4096",
+          "midtrain-schedule",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "f54a41c1b7b43ef4",
+      "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "f54a41c1b7b43ef4",
+    "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86521538,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86521538,
+    "mixture": {
+      "cursors": {
+        "original": {
+          "file_idx": 7,
+          "pos": 86521538,
+          "epoch": 1,
+          "pq_idx": 7,
+          "rg_idx": 86521538
+        },
+        "midtrain_r30": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 786497536,
+      "source_tokens": {
+        "original": 786497536,
+        "midtrain_r30": 0,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 0
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1534062642313452,
+    "smooth_train_loss": 3.3564800354053057,
+    "total_training_time": 4609.387755393982,
+    "stage_training_flops": 875763049955328000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 875763049955328000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002000.json
new file mode 100644
index 0000000000000000000000000000000000000000..0bd69ef52ec604d9dac8540f7af1fda94f1d8cf0
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002000.json
@@ -0,0 +1,222 @@
+{
+  "step": 2000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "val_bpb": 1.1278678135192355,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r12-ctx4096",
+    "wandb_run_id": "e564d552",
+    "wandb_group": "clean1930s-d12-ctx4096",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": 2520,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": 1000,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok",
+    "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r12-ctx4096",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "23031c7d2f7cc08da131639554d952aef02069e9",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r12-ctx4096",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r12-ctx4096",
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 180,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 78,
+          "num_train_shards": 40,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 101,
+          "num_train_shards": 20,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 1321205760,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 924844032,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 1189085184,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 524288,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r12-ctx4096",
+        "group": "clean1930s-d12-ctx4096",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio12",
+          "ctx4096",
+          "midtrain-schedule",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "f54a41c1b7b43ef4",
+      "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "f54a41c1b7b43ef4",
+    "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 1,
+    "pos": 23801282,
+    "epoch": 1,
+    "pq_idx": 1,
+    "rg_idx": 23801282,
+    "mixture": {
+      "cursors": {
+        "original": {
+          "file_idx": 9,
+          "pos": 24872256,
+          "epoch": 1,
+          "pq_idx": 9,
+          "rg_idx": 24872256
+        },
+        "midtrain_r30": {
+          "file_idx": 1,
+          "pos": 23801282,
+          "epoch": 1,
+          "pq_idx": 1,
+          "rg_idx": 23801282
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 1048641536,
+      "source_tokens": {
+        "original": 924844032,
+        "midtrain_r30": 123797504,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 1
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1278678135192355,
+    "smooth_train_loss": 3.2400561803845425,
+    "total_training_time": 6135.895956039429,
+    "stage_training_flops": 1167684066607104000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1167684066607104000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002500.json b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002500.json
new file mode 100644
index 0000000000000000000000000000000000000000..a1780493a2f25f22224217df8825171816c48374
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002500.json
@@ -0,0 +1,222 @@
+{
+  "step": 2500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "val_bpb": 1.0910322707751114,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r12-ctx4096",
+    "wandb_run_id": "e564d552",
+    "wandb_group": "clean1930s-d12-ctx4096",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": 2520,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": 1000,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok",
+    "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r12-ctx4096",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "23031c7d2f7cc08da131639554d952aef02069e9",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r12-ctx4096",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r12-ctx4096",
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 180,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 78,
+          "num_train_shards": 40,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 101,
+          "num_train_shards": 20,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 1321205760,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 924844032,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 1189085184,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 524288,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r12-ctx4096",
+        "group": "clean1930s-d12-ctx4096",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio12",
+          "ctx4096",
+          "midtrain-schedule",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "f54a41c1b7b43ef4",
+      "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "f54a41c1b7b43ef4",
+    "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 1,
+    "pos": 21704066,
+    "epoch": 1,
+    "pq_idx": 1,
+    "rg_idx": 21704066,
+    "mixture": {
+      "cursors": {
+        "original": {
+          "file_idx": 9,
+          "pos": 24872256,
+          "epoch": 1,
+          "pq_idx": 9,
+          "rg_idx": 24872256
+        },
+        "midtrain_r30": {
+          "file_idx": 2,
+          "pos": 64249216,
+          "epoch": 1,
+          "pq_idx": 2,
+          "rg_idx": 64249216
+        },
+        "midtrain_r60": {
+          "file_idx": 1,
+          "pos": 21704066,
+          "epoch": 1,
+          "pq_idx": 1,
+          "rg_idx": 21704066
+        }
+      },
+      "cumulative_tokens": 1310785536,
+      "source_tokens": {
+        "original": 924844032,
+        "midtrain_r30": 264241152,
+        "midtrain_r60": 121700352
+      },
+      "active_stage_idx": 2
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0910322707751114,
+    "smooth_train_loss": 3.1627455750850237,
+    "total_training_time": 7662.492534160614,
+    "stage_training_flops": 1459605083258880000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1459605083258880000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002520.json b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002520.json
new file mode 100644
index 0000000000000000000000000000000000000000..4398eed8665ad952f4718e0d6d8070b7e5b73feb
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/meta_002520.json
@@ -0,0 +1,222 @@
+{
+  "step": 2520,
+  "training_complete": true,
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "val_bpb": 1.0901240605951916,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d12-r12-ctx4096",
+    "wandb_run_id": "e564d552",
+    "wandb_group": "clean1930s-d12-ctx4096",
+    "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": 2520,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": 1000,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok",
+    "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints",
+    "experiment_id": "clean1930s-d12-r12-ctx4096",
+    "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r12-ctx4096/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "23031c7d2f7cc08da131639554d952aef02069e9",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d12-r12-ctx4096",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d12-r12-ctx4096",
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 180,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 78,
+          "num_train_shards": 40,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 101,
+          "num_train_shards": 20,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 1321205760,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 924844032,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 1189085184,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 524288,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d12-r12-ctx4096",
+        "group": "clean1930s-d12-ctx4096",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d12",
+          "ratio12",
+          "ctx4096",
+          "midtrain-schedule",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "f54a41c1b7b43ef4",
+      "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "f54a41c1b7b43ef4",
+    "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 1,
+    "pos": 32190146,
+    "epoch": 1,
+    "pq_idx": 1,
+    "rg_idx": 32190146,
+    "mixture": {
+      "cursors": {
+        "original": {
+          "file_idx": 9,
+          "pos": 24872256,
+          "epoch": 1,
+          "pq_idx": 9,
+          "rg_idx": 24872256
+        },
+        "midtrain_r30": {
+          "file_idx": 2,
+          "pos": 64249216,
+          "epoch": 1,
+          "pq_idx": 2,
+          "rg_idx": 64249216
+        },
+        "midtrain_r60": {
+          "file_idx": 1,
+          "pos": 32190146,
+          "epoch": 1,
+          "pq_idx": 1,
+          "rg_idx": 32190146
+        }
+      },
+      "cumulative_tokens": 1321271296,
+      "source_tokens": {
+        "original": 924844032,
+        "midtrain_r30": 264241152,
+        "midtrain_r60": 132186112
+      },
+      "active_stage_idx": 2
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0901240605951916,
+    "smooth_train_loss": 3.1538503603170533,
+    "total_training_time": 7723.430352926254,
+    "stage_training_flops": 1471281923924951040,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1471281923924951040
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cf7ecd87b045a3000232070ebc4590135d45489
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a02d598c92e1ad5b765d3d0eb420db85489ea39645054c5fa07b5c1064a0d98
+size 792761690
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4436a9d653f8c39698de78339eec870008e802f
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76925cc051b220fef8022928c1be79ff33cc8c645bac3d9c99330a5cd0481b4f
+size 792761690
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7feb80a0904e958ca32da58c0e398f8c744a5849
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae4ba119454fbb2e5a9002c6b085cf6defd58e9e56d0020b1afe48e651ceced8
+size 792761690
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..313f8416c37c4600b040ce20b0111ffca2e5f0ab
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8f34df504598c38686c112f21609ef2c4782e1a2a9d49e752f5eee461f873ba
+size 792761690
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002500.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..547436e6f79d03a4bd7ba2927d0fc00b37538efd
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ec550702fc1f867fe73c310471c6a1c1bcfaa11dd2384c9276d6c84f389c90
+size 792761690
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002520.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002520.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dfb4d65d9679408f33e4a4cf39bb64e165de475c
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/model_002520.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b15ba4053c9c3165988bd8bb70adae3937402f8d8b63c3c592d8b1619de2ee9b
+size 792761690
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_000500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86a28e94665555ad5b6b5cdab78b7dfec2ae711a
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_000500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d99ad5c617d963fd673cd81134e61bdc8163dbd86a268357ad18f6eb68803dfd
+size 1246165357
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_001000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..66c3a6b28275cd7da3f10db8a9eb8283edeb536a
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_001000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c3efcacd3be64a437fb00c2d8d58ffece9581d9e3f8f53bcfba41d762ead912
+size 1246165357
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_001500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ff49188e5e47d1b50f5d29e71bc4ca49a9b931db
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_001500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:969e98e437d5e82ec160b8bac637a4f50efc79f622ef4c86328c70a38a7471fd
+size 1246165357
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9459da8971afae1f34f00ce4465c995d40e5e904
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7139630f8def9773169031966aa4c00e2daf5f66d30a83685e92368f0a6e60ff
+size 1246165357
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002500_rank0.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f65013e9aae7e61e655ffc355e1f3c7f06432a3f
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f40de772fe8dab02d5de78f4c260b56321424a91c3d24bfc5fb095590bff9d61
+size 1246165357
diff --git a/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002520_rank0.pt b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002520_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c4555c974380a6c1d1a33210a0105433d2803831
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/base_checkpoints/optim_002520_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69b60e67ecc3b8ab39ccb6f76883c53e6e1546c4c3484a77f32e8fa33095eb36
+size 1246165357
diff --git a/experiments/clean1930s-d12-r12-ctx4096/config.json b/experiments/clean1930s-d12-r12-ctx4096/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d0b1b7a1995474ac810eb41efe80d1d1c137f03c
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/config.json
@@ -0,0 +1,105 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "datasets": {
+    "original": {
+      "adapter": "parquet_shards",
+      "repo": "jbduran/think-dataset-clean-1930s",
+      "revision": "main",
+      "validation_shard": 472,
+      "num_train_shards": 180,
+      "download_workers": 4
+    },
+    "midtrain_r30": {
+      "adapter": "parquet_shards",
+      "repo": "zachnorton03/think-midtrain",
+      "revision": "main",
+      "subfolder": "mixed/ratio_30/data",
+      "validation_shard": 78,
+      "num_train_shards": 40,
+      "download_workers": 4
+    },
+    "midtrain_r60": {
+      "adapter": "parquet_shards",
+      "repo": "zachnorton03/think-midtrain",
+      "revision": "main",
+      "subfolder": "mixed/ratio_60/data",
+      "validation_shard": 101,
+      "num_train_shards": 20,
+      "download_workers": 4
+    }
+  },
+  "mixture_schedule": {
+    "total_tokens": 1321205760,
+    "seed_data": 1234,
+    "max_epochs": 4,
+    "stages": [
+      {
+        "name": "base",
+        "start_tokens": 0,
+        "source": "original"
+      },
+      {
+        "name": "injection",
+        "start_tokens": 924844032,
+        "source": "midtrain_r30"
+      },
+      {
+        "name": "decay_mix",
+        "start_tokens": 1189085184,
+        "source": "midtrain_r60"
+      }
+    ]
+  },
+  "tokenizer": {
+    "mode": "reuse",
+    "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "require_no_wrap": true,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 12,
+    "scaling_params": 110100912,
+    "target_param_data_ratio": 12,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "device_batch_size": 8,
+    "total_batch_size": 524288,
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "seed": 42,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano",
+    "keep_local_checkpoints": 1
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "clean1930s-d12-r12-ctx4096",
+    "group": "clean1930s-d12-ctx4096",
+    "tags": [
+      "think-dataset-clean-1930s",
+      "d12",
+      "ratio12",
+      "ctx4096",
+      "midtrain-schedule",
+      "fp8"
+    ]
+  },
+  "config_fingerprint": "f54a41c1b7b43ef4",
+  "artifact_path": "experiments/clean1930s-d12-r12-ctx4096"
+}
diff --git a/experiments/clean1930s-d12-r12-ctx4096/run.json b/experiments/clean1930s-d12-r12-ctx4096/run.json
new file mode 100644
index 0000000000000000000000000000000000000000..423e261c0c21af853f3b23fd29c3a99f215a680f
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/run.json
@@ -0,0 +1,10 @@
+{
+  "experiment_id": "clean1930s-d12-r12-ctx4096",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d12-r12-ctx4096",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "f54a41c1b7b43ef4",
+  "wandb_run_id": "e564d552",
+  "created_at": 1784920582
+}
diff --git a/experiments/clean1930s-d12-r12-ctx4096/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r12-ctx4096/tokenizer/experiment_tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/tokenizer/experiment_tokenizer.json
@@ -0,0 +1,18 @@
+{
+  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 200,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 1000000000000,
+    "doc_cap": 1000000000,
+    "vocab_size": 32768
+  },
+  "created_at": 1784129374
+}
diff --git a/experiments/clean1930s-d12-r12-ctx4096/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r12-ctx4096/tokenizer/token_bytes.pt
new file mode 100644
index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/tokenizer/token_bytes.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2
+size 132649
diff --git a/experiments/clean1930s-d12-r12-ctx4096/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r12-ctx4096/tokenizer/tokenizer.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac
--- /dev/null
+++ b/experiments/clean1930s-d12-r12-ctx4096/tokenizer/tokenizer.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f
+size 410542
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/meta_000500.json
new file mode 100644
index 0000000000000000000000000000000000000000..01421c0127a8faa5264a9a81f35d49f347ecb8bf
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/meta_000500.json
@@ -0,0 +1,145 @@
+{
+  "step": 500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "val_bpb": 1.1871477110433892,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "wandb_run_id": "14f8d6ff",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "97f9a8373c6852c594912151249caf4b8a9ee998",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 200,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 1000000000000,
+        "doc_cap": 1000000000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "L",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "67b81539f6613d04",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "67b81539f6613d04"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24322769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24322769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1871477110433892,
+    "smooth_train_loss": 3.3945704550936946,
+    "total_training_time": 999.2132678031921,
+    "stage_training_flops": 3245763761012736000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 3245763761012736000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/model_000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..705d7e6c238a2cef6a6e113a35be7f1f4e5dfdc6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/model_000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54be6eedde558d22722ce09563434b9890e9702841b45b1b698f8ac33daa7025
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6019132406a0f07ed7e34992c9511d9ea162817
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7766244d86d99a2cad1b9f196413a0e193213635e2f6bc85ae91a72fec023f4
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9140b8401d949264927d1f48a260ba7804dfebc4
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85ad482372c1535df095998bf353351c17eaa2e5a7002141025c9405167de291
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4117c07abcb1237709d458625c0b82bb92727fba
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc264d206ff72630f771afe9f7c8faff53f5367b27b2134baa05b9390daf3d03
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9992d0354940b9ac6d9ad26b0aa36fe0d952c12e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac1bfb557caf498b940743b64a242b891c7d9744301c14214369c15990734781
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank4.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank4.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc6a24351caf8478c93d63a56c0dc77b782a25a9
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank4.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1020ca5e6e2c633a755a5c49e9d38c5bf560ba4482f6db2634ac11f78daf8793
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank5.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank5.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8d76b86d2353d861239b5c77c292f5d6f63a2faf
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank5.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bbcbfa19ab1f94f0b4e5bba16ec95e31c342b55744701f1f9e2d05baba443a7
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank6.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank6.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cc83cd5d82ff73432754e7dafc63e09957b901ae
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank6.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f80a5b1ebe129b2e08eaaa61afb84960acad1b07e1d19987f488fec6c43d7350
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank7.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank7.pt
new file mode 100644
index 0000000000000000000000000000000000000000..44bff9e9d2a7fea38b927bcffc1fc6f122d3d1e5
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/base_checkpoints/optim_000500_rank7.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7d2640548eb206eeaf08804b69281500de1307810847c75cdfb3bb62a694872
+size 717465265
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/config.json b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7ac24fafc268f4fcf39ab10ee855c45d01763f6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/config.json
@@ -0,0 +1,62 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 200,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 1000000000000,
+    "doc_cap": 1000000000,
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "require_no_wrap": true,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 24,
+    "scaling_params": 729810624,
+    "target_param_data_ratio": 12,
+    "max_seq_len": 4096,
+    "window_pattern": "L",
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "seed": 42,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano"
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "group": "clean1930s-d24",
+    "tags": [
+      "think-dataset-clean-1930s",
+      "d24",
+      "ratio12",
+      "ctx4096",
+      "fp8"
+    ]
+  },
+  "config_fingerprint": "67b81539f6613d04",
+  "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-fulltok-v1"
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/run.json b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/run.json
new file mode 100644
index 0000000000000000000000000000000000000000..a47fef12476f81783db90438111e383b73812ca5
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/run.json
@@ -0,0 +1,10 @@
+{
+  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "67b81539f6613d04",
+  "wandb_run_id": "14f8d6ff",
+  "created_at": 1784129314
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/experiment_tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/experiment_tokenizer.json
@@ -0,0 +1,18 @@
+{
+  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 200,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 1000000000000,
+    "doc_cap": 1000000000,
+    "vocab_size": 32768
+  },
+  "created_at": 1784129374
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/token_bytes.pt b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/token_bytes.pt
new file mode 100644
index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/token_bytes.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2
+size 132649
diff --git a/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/tokenizer.pkl b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/tokenizer.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-fulltok-v1/tokenizer/tokenizer.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f
+size 410542
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_006500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_006500.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a9c4f263598619e5dcb93f6cdbc43e7de15b603
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_006500.json
@@ -0,0 +1,231 @@
+{
+  "step": 6500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "val_bpb": 0.9247502060081154,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "wandb_run_id": "d0cda484",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8,midtrain-schedule,mix-og,0-30-60",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": 8352,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "init_from_checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "init_from_step": 6000,
+    "no_init_optimizer": false,
+    "branch_lr_schedule": "continue",
+    "branch_parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok",
+    "mixture_source_dirs": "{\"midtrain_r30\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "640d2e56d9ca333095a9e23dcf54d498341552e8",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+      "branch": {
+        "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "parent_step": 6000,
+        "lr_schedule": "continue",
+        "load_optimizer": true
+      },
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 140,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 1262,
+          "num_train_shards": 170,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 523,
+          "num_train_shards": 90,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 8757706752,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 6129975296,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 7881097216,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8",
+          "midtrain-schedule",
+          "mix-og",
+          "0-30-60"
+        ]
+      },
+      "config_fingerprint": "43f2fdfbde0b68d4",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_checkpoint_step": 6000,
+    "config_fingerprint": "43f2fdfbde0b68d4"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24423073,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24423073,
+    "mixture": {
+      "cursors": {
+        "midtrain_r30": {
+          "file_idx": 5,
+          "pos": 24423073,
+          "epoch": 1,
+          "pq_idx": 5,
+          "rg_idx": 24423073
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 6815875072,
+      "source_tokens": {
+        "original": 0,
+        "midtrain_r30": 524419072,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 1
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9247502060081154,
+    "smooth_train_loss": 2.6664623196737285,
+    "total_training_time": 1108.3150436878204,
+    "stage_start_step": 6000,
+    "stage_training_flops": 2711401109913600000,
+    "inherited_parent_flops": 3.25368133189632e+19,
+    "cumulative_pipeline_training_flops": 3.52482144288768e+19
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_007000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_007000.json
new file mode 100644
index 0000000000000000000000000000000000000000..e63c027736cd78a791fe6acc1714288093e4ace3
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_007000.json
@@ -0,0 +1,231 @@
+{
+  "step": 7000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "val_bpb": 0.9164157458762685,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "wandb_run_id": "d0cda484",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8,midtrain-schedule,mix-og,0-30-60",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": 8352,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "init_from_checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "init_from_step": 6000,
+    "no_init_optimizer": false,
+    "branch_lr_schedule": "continue",
+    "branch_parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok",
+    "mixture_source_dirs": "{\"midtrain_r30\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "640d2e56d9ca333095a9e23dcf54d498341552e8",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+      "branch": {
+        "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "parent_step": 6000,
+        "lr_schedule": "continue",
+        "load_optimizer": true
+      },
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 140,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 1262,
+          "num_train_shards": 170,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 523,
+          "num_train_shards": 90,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 8757706752,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 6129975296,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 7881097216,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8",
+          "midtrain-schedule",
+          "mix-og",
+          "0-30-60"
+        ]
+      },
+      "config_fingerprint": "43f2fdfbde0b68d4",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_checkpoint_step": 6000,
+    "config_fingerprint": "43f2fdfbde0b68d4"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48715073,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48715073,
+    "mixture": {
+      "cursors": {
+        "midtrain_r30": {
+          "file_idx": 10,
+          "pos": 48715073,
+          "epoch": 1,
+          "pq_idx": 10,
+          "rg_idx": 48715073
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 7340163072,
+      "source_tokens": {
+        "original": 0,
+        "midtrain_r30": 1048707072,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 1
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9164157458762685,
+    "smooth_train_loss": 2.6353370917159102,
+    "total_training_time": 2243.1743581295013,
+    "stage_start_step": 6000,
+    "stage_training_flops": 5422802219827200000,
+    "inherited_parent_flops": 3.25368133189632e+19,
+    "cumulative_pipeline_training_flops": 3.79596155387904e+19
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_007500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_007500.json
new file mode 100644
index 0000000000000000000000000000000000000000..4473a8c6fcd1e158a0836a12b326138f0f82634a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_007500.json
@@ -0,0 +1,231 @@
+{
+  "step": 7500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "val_bpb": 0.9050723402326186,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "wandb_run_id": "d0cda484",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8,midtrain-schedule,mix-og,0-30-60",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": 8352,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "init_from_checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "init_from_step": 6000,
+    "no_init_optimizer": false,
+    "branch_lr_schedule": "continue",
+    "branch_parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok",
+    "mixture_source_dirs": "{\"midtrain_r30\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "640d2e56d9ca333095a9e23dcf54d498341552e8",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+      "branch": {
+        "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "parent_step": 6000,
+        "lr_schedule": "continue",
+        "load_optimizer": true
+      },
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 140,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 1262,
+          "num_train_shards": 170,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 523,
+          "num_train_shards": 90,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 8757706752,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 6129975296,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 7881097216,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8",
+          "midtrain-schedule",
+          "mix-og",
+          "0-30-60"
+        ]
+      },
+      "config_fingerprint": "43f2fdfbde0b68d4",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_checkpoint_step": 6000,
+    "config_fingerprint": "43f2fdfbde0b68d4"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 15,
+    "pos": 73007073,
+    "epoch": 1,
+    "pq_idx": 15,
+    "rg_idx": 73007073,
+    "mixture": {
+      "cursors": {
+        "midtrain_r30": {
+          "file_idx": 15,
+          "pos": 73007073,
+          "epoch": 1,
+          "pq_idx": 15,
+          "rg_idx": 73007073
+        },
+        "midtrain_r60": {
+          "file_idx": 0,
+          "pos": 0,
+          "epoch": 1,
+          "pq_idx": 0,
+          "rg_idx": 0
+        }
+      },
+      "cumulative_tokens": 7864451072,
+      "source_tokens": {
+        "original": 0,
+        "midtrain_r30": 1572995072,
+        "midtrain_r60": 0
+      },
+      "active_stage_idx": 1
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9050723402326186,
+    "smooth_train_loss": 2.64315452719698,
+    "total_training_time": 3379.8451437950134,
+    "stage_start_step": 6000,
+    "stage_training_flops": 8134203329740800000,
+    "inherited_parent_flops": 3.25368133189632e+19,
+    "cumulative_pipeline_training_flops": 4.0671016648704e+19
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_008000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_008000.json
new file mode 100644
index 0000000000000000000000000000000000000000..595b1ca9d831b8eb81c200cc0c4a4764d476d48a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_008000.json
@@ -0,0 +1,231 @@
+{
+  "step": 8000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "val_bpb": 0.890866525025072,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "wandb_run_id": "d0cda484",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8,midtrain-schedule,mix-og,0-30-60",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": 8352,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "init_from_checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "init_from_step": 6000,
+    "no_init_optimizer": false,
+    "branch_lr_schedule": "continue",
+    "branch_parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok",
+    "mixture_source_dirs": "{\"midtrain_r30\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "640d2e56d9ca333095a9e23dcf54d498341552e8",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+      "branch": {
+        "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "parent_step": 6000,
+        "lr_schedule": "continue",
+        "load_optimizer": true
+      },
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 140,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 1262,
+          "num_train_shards": 170,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 523,
+          "num_train_shards": 90,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 8757706752,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 6129975296,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 7881097216,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8",
+          "midtrain-schedule",
+          "mix-og",
+          "0-30-60"
+        ]
+      },
+      "config_fingerprint": "43f2fdfbde0b68d4",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_checkpoint_step": 6000,
+    "config_fingerprint": "43f2fdfbde0b68d4"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 7645729,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 7645729,
+    "mixture": {
+      "cursors": {
+        "midtrain_r30": {
+          "file_idx": 15,
+          "pos": 89653344,
+          "epoch": 1,
+          "pq_idx": 15,
+          "rg_idx": 89653344
+        },
+        "midtrain_r60": {
+          "file_idx": 5,
+          "pos": 7645729,
+          "epoch": 1,
+          "pq_idx": 5,
+          "rg_idx": 7645729
+        }
+      },
+      "cumulative_tokens": 8388739072,
+      "source_tokens": {
+        "original": 0,
+        "midtrain_r30": 1589641216,
+        "midtrain_r60": 507641856
+      },
+      "active_stage_idx": 2
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 0.890866525025072,
+    "smooth_train_loss": 2.5129958853985905,
+    "total_training_time": 4517.323269367218,
+    "stage_start_step": 6000,
+    "stage_training_flops": 10845604439654400000,
+    "inherited_parent_flops": 3.25368133189632e+19,
+    "cumulative_pipeline_training_flops": 4.33824177586176e+19
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_008352.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_008352.json
new file mode 100644
index 0000000000000000000000000000000000000000..28eba485a72455cdea76a662c5507efdf9c824cd
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/meta_008352.json
@@ -0,0 +1,231 @@
+{
+  "step": 8352,
+  "training_complete": true,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "val_bpb": 0.8864625921976007,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "wandb_run_id": "d0cda484",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8,midtrain-schedule,mix-og,0-30-60",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": 8352,
+    "target_flops": -1.0,
+    "target_param_data_ratio": -1.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "init_from_checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "init_from_step": 6000,
+    "no_init_optimizer": false,
+    "branch_lr_schedule": "continue",
+    "branch_parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok",
+    "mixture_source_dirs": "{\"midtrain_r30\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/pretok_midtrain_r60\"}",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "640d2e56d9ca333095a9e23dcf54d498341552e8",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+      "branch": {
+        "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "parent_step": 6000,
+        "lr_schedule": "continue",
+        "load_optimizer": true
+      },
+      "datasets": {
+        "original": {
+          "adapter": "parquet_shards",
+          "repo": "jbduran/think-dataset-clean-1930s",
+          "revision": "main",
+          "validation_shard": 472,
+          "num_train_shards": 140,
+          "download_workers": 4
+        },
+        "midtrain_r30": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_30/data",
+          "validation_shard": 1262,
+          "num_train_shards": 170,
+          "download_workers": 4
+        },
+        "midtrain_r60": {
+          "adapter": "parquet_shards",
+          "repo": "zachnorton03/think-midtrain",
+          "revision": "main",
+          "subfolder": "mixed/ratio_60/data",
+          "validation_shard": 523,
+          "num_train_shards": 90,
+          "download_workers": 4
+        }
+      },
+      "mixture_schedule": {
+        "total_tokens": 8757706752,
+        "seed_data": 1234,
+        "max_epochs": 4,
+        "stages": [
+          {
+            "name": "base",
+            "start_tokens": 0,
+            "source": "original"
+          },
+          {
+            "name": "injection",
+            "start_tokens": 6129975296,
+            "source": "midtrain_r30"
+          },
+          {
+            "name": "decay_mix",
+            "start_tokens": 7881097216,
+            "source": "midtrain_r60"
+          }
+        ]
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8",
+          "midtrain-schedule",
+          "mix-og",
+          "0-30-60"
+        ]
+      },
+      "config_fingerprint": "43f2fdfbde0b68d4",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_checkpoint_step": 6000,
+    "config_fingerprint": "43f2fdfbde0b68d4"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 8,
+    "pos": 76747297,
+    "epoch": 1,
+    "pq_idx": 8,
+    "rg_idx": 76747297,
+    "mixture": {
+      "cursors": {
+        "midtrain_r30": {
+          "file_idx": 15,
+          "pos": 89653344,
+          "epoch": 1,
+          "pq_idx": 15,
+          "rg_idx": 89653344
+        },
+        "midtrain_r60": {
+          "file_idx": 8,
+          "pos": 76747297,
+          "epoch": 1,
+          "pq_idx": 8,
+          "rg_idx": 76747297
+        }
+      },
+      "cumulative_tokens": 8757837824,
+      "source_tokens": {
+        "original": 0,
+        "midtrain_r30": 1589641216,
+        "midtrain_r60": 876740608
+      },
+      "active_stage_idx": 2
+    }
+  },
+  "loop_state": {
+    "min_val_bpb": 0.8864625921976007,
+    "smooth_train_loss": 2.446982447319594,
+    "total_training_time": 5318.38954949379,
+    "stage_start_step": 6000,
+    "stage_training_flops": 12754430821033574400,
+    "inherited_parent_flops": 3.25368133189632e+19,
+    "cumulative_pipeline_training_flops": 4.5291244139996774e+19
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_006500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_006500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7d64b883be4b64dda220575ef9f19f397026a9f2
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_006500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b164ffcc7be627e3afb0b0e8ae0a5ff3b97a46f8aa01c9d37f3b9c3b004b572
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_007000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_007000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ec2998bc6ed84a31b1f9e60ba00615098e9ea47e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_007000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fe7a666b22db61b0eb89da7b07ea90fd606b31fdbe6e2d665cea4e09a54dd40
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_007500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_007500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af3763aa61cb6f03f7984e8b482804688d85219d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_007500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0abeff0941358f864da576247a97de9d9c7ae4d00e5e4e08115b32fe0408bdeb
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_008000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_008000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5c8f847a906799ccb3dd644b1b4a536ae41b72e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_008000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0950335e48894bc28488bab0a27c7bae5fa49389290a729216f12da88db49658
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_008352.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_008352.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a5eab61729e3603eb233898880161ff421c8366
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/model_008352.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd2e57b069247cb6eaebbbf6e8acc584b8b8d94d30910bc7cd989210424af1a1
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5405851816cb69929b28cd77276c22b11cf05200
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8222dfefc5520aaf6cb5914daf96f97bc86b176bd055c9d737890e84c9e960ef
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0435772226add041a3b5ca9178e66809e8085bff
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d11a80ba154d4927cacd66ddc6059fc6408944bf484fc1a37536db6ef974dfce
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..27675a76e3d72e3c8dd877862ab12f7cf47a8e6e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ab49eacd43bbc4f3c9949fcd37cd84118aedf94a8449bdf4536bd280f540d0
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b8ee303b71378e7eb49a995c20d4d8a93283b16a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_006500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68317bde289fc05e38737cab8ed897f0fe3555da32ac1d45ae947995aff526ee
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3e833844022d495cce35a2e36016ecead56b791
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d8962c430aa9630a40edb89836363111497e56b521bed8290770d242d86e576
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..545bbd7ae49b33e99ea0b3862025ca23d67d4e30
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922d166bb739a0b321085bec48de5f1ff1bf6e58ac805ad50dd4ec59282e905f
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af05d9a7175251eca64dc5de87c376085a5b51ca
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e421c62a57d07c517a6365e31cbdfa87806c9669a25bc12afb19110db86af537
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b2ee4404eedd6f20fdd4e63cf2875257aef70d9
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007000_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3676560bd5ee261a1d30ef5b0a0309827486d043b527de6989327bd8b5b45421
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11f2339a00671704b5ad3c619840db7406da573f
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b25117908bf46c2401d78a8a623ee44e859c4b6af761bcaa9daf98e73c08c9e9
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..931943c3f119d407074261851836f438e13e8ac7
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f56c152f095b226b639b11dc2ec0ac95ba5a7601f4493f6375dc5083c6729e
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c5e48632468fd20851d558886fc033474ab54cbf
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d64e27870d1701eeeb5e6ccf41524b06eba86ecb1df13683ff65a02287bf684f
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9bdedba4c705f4f5a4498fb4456029a2d27f6a4d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_007500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98925db0bfd97e4c45e5bc2d6d2dbaca5eabdbbb12a307eed6080311d70d7e05
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..668ab4d9a6d88595f1fedacc8c769c7700c87733
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a85181a8513387352fbb089dc17bf8828d1d4ebac47c49d173605edf1a99fe8
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6edad79d101e0cf0ada0072d715a47b6f487083d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de5c5e81af359caf49287a7f5cf812ed31565adfc9c2e3d13c793654368dd20a
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a08560632eefa985e2016094465bf9d22301601d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae5e83b2b872caf16699e319fe8e79d1edeeade57e087446667530bf2573ae40
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8dd26146feaadb03d14f20f3c3480eea6c867f0
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008000_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc19e7de51b183fe37e870d41f54ac4bd0916cea69427867795d4e7dd77dc3e9
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab66e9738a2332cd155d820800f211d34dad6ed6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da8d750253b41f3f4e88db1f7a81d864f3a6c8704503860e5878a4af78a4b91a
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee2ccb0f2a8baf0464aba69616470864efcef14b
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31abfbbfdf30576fc9cf8be6f4b707ee9770806a4cee172d62128a60f03ee8b8
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..588950580579d1a8ec53748470392943976314dc
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e35d122f69594d1606daf881332ec492fd5cf7b70f63ab0ea0f2530222b1c9c9
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea767da291edb82174cce399e8b5fa3a6fd83719
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/base_checkpoints/optim_008352_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:253eeb5478238b4f3df442186ca5518d99cb8089082e3b79b70c95b95c30adcb
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e95e6c6abcca3d11902c247a39300f6e852ced7
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/config.json
@@ -0,0 +1,114 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "branch": {
+    "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_step": 6000,
+    "lr_schedule": "continue",
+    "load_optimizer": true
+  },
+  "datasets": {
+    "original": {
+      "adapter": "parquet_shards",
+      "repo": "jbduran/think-dataset-clean-1930s",
+      "revision": "main",
+      "validation_shard": 472,
+      "num_train_shards": 140,
+      "download_workers": 4
+    },
+    "midtrain_r30": {
+      "adapter": "parquet_shards",
+      "repo": "zachnorton03/think-midtrain",
+      "revision": "main",
+      "subfolder": "mixed/ratio_30/data",
+      "validation_shard": 1262,
+      "num_train_shards": 170,
+      "download_workers": 4
+    },
+    "midtrain_r60": {
+      "adapter": "parquet_shards",
+      "repo": "zachnorton03/think-midtrain",
+      "revision": "main",
+      "subfolder": "mixed/ratio_60/data",
+      "validation_shard": 523,
+      "num_train_shards": 90,
+      "download_workers": 4
+    }
+  },
+  "mixture_schedule": {
+    "total_tokens": 8757706752,
+    "seed_data": 1234,
+    "max_epochs": 4,
+    "stages": [
+      {
+        "name": "base",
+        "start_tokens": 0,
+        "source": "original"
+      },
+      {
+        "name": "injection",
+        "start_tokens": 6129975296,
+        "source": "midtrain_r30"
+      },
+      {
+        "name": "decay_mix",
+        "start_tokens": 7881097216,
+        "source": "midtrain_r60"
+      }
+    ]
+  },
+  "tokenizer": {
+    "mode": "reuse",
+    "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "require_no_wrap": true,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 24,
+    "scaling_params": 729810624,
+    "target_param_data_ratio": 12,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "seed": 42,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano",
+    "keep_local_checkpoints": 1
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+    "group": "clean1930s-d24",
+    "tags": [
+      "think-dataset-clean-1930s",
+      "d24",
+      "ratio12",
+      "ctx4096",
+      "sssl",
+      "fp8",
+      "midtrain-schedule",
+      "mix-og",
+      "0-30-60"
+    ]
+  },
+  "config_fingerprint": "43f2fdfbde0b68d4",
+  "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og"
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/core.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/core.json
new file mode 100644
index 0000000000000000000000000000000000000000..8163192ced4098cd8ac479fd15b5d479009d7703
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/core.json
@@ -0,0 +1,56 @@
+{
+  "model": "base_model (step 8352)",
+  "step": 8352,
+  "bpb": {},
+  "core_metric": 0.14018200582682225,
+  "core_results": {
+    "hellaswag_zeroshot": 0.33041226863861084,
+    "jeopardy": 0.012281530536711216,
+    "bigbench_qa_wikidata": 0.24049012362957,
+    "arc_easy": 0.41456228494644165,
+    "arc_challenge": 0.239761084318161,
+    "copa": 0.5999999642372131,
+    "commonsense_qa": 0.32514333724975586,
+    "piqa": 0.5854189395904541,
+    "openbook_qa": 0.26200002431869507,
+    "lambada_openai": 0.3720162808895111,
+    "hellaswag": 0.33379805088043213,
+    "winograd": 0.6410256624221802,
+    "winogrande": 0.5177584886550903,
+    "bigbench_dyck_languages": 0.13700000941753387,
+    "agi_eval_lsat_ar": 0.2652173936367035,
+    "bigbench_cs_algorithms": 0.396212100982666,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.14124882221221924,
+    "coqa": 0.18476763367652893,
+    "boolq": 0.5993883609771729,
+    "bigbench_language_identification": 0.25679999589920044
+  },
+  "centered_results": {
+    "hellaswag_zeroshot": 0.10721635818481445,
+    "jeopardy": 0.012281530536711216,
+    "bigbench_qa_wikidata": 0.24049012362957,
+    "arc_easy": 0.21941637992858887,
+    "arc_challenge": -0.013651887575785318,
+    "copa": 0.19999992847442627,
+    "commonsense_qa": 0.1564291715621948,
+    "piqa": 0.1708378791809082,
+    "openbook_qa": 0.016000032424926758,
+    "lambada_openai": 0.3720162808895111,
+    "hellaswag": 0.11173073450724284,
+    "winograd": 0.28205132484436035,
+    "winogrande": 0.035516977310180664,
+    "bigbench_dyck_languages": 0.13700000941753387,
+    "agi_eval_lsat_ar": 0.08152174204587935,
+    "bigbench_cs_algorithms": 0.396212100982666,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.14124882221221924,
+    "coqa": 0.18476763367652893,
+    "boolq": -0.054241155323229324,
+    "bigbench_language_identification": 0.18239823531265176
+  },
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/samples.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/samples.json
new file mode 100644
index 0000000000000000000000000000000000000000..a6a2dcea808b8ffe4ac471142bad747cfa64fc0d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/samples.json
@@ -0,0 +1,48 @@
+{
+  "model": "base_model (step 8352)",
+  "step": 8352,
+  "bpb": {},
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of France.\n\nThe capital of France is the capital of France.\n\nThe"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is\n\nA. 1. 2. 3. 4. "
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Sunday.\n\nThe Sunday after the Friday, the Sunday after the Sunday, the Sunday"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is cold, and vice versa.\n\nThe opposite of cold is hot, and vice versa"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: Mercury, Venus, Mars, Jupiter, Saturn, Uranus, Neptune,"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is a deep, rich, velvety crimson, with a large, full,"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of times 5 is contained in 13, and 3 is"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>per anus ullams v .. niteen erl\n\nSAC OF THE UNKNOWN MEASURES\n\nJustifiable Xiwerermarg \u00abnsng van Fairie half Psos 1n FSA || Y | ||| -_|||| || | || || | |||_ | | | || | ||| |}| |_|_|_| || |_|| |) | || | | | | | _| ||| | |\n\nai i Vi : } J | | |",
+    "<|bos|>Lady Encurt, bi le Scotch wu cften mud sbee, for Lady Percrash Claveringastes forlorn she hard, fine or dime he ran ahead of work, veile, tat Nells under the wualdkhorns wan garee mad dulep, Woman Asking, he one see, weeded som dulep a way South Unionalysnowed bruts arayn made him kyill, sic lass tat Brappy Beuld sco declared review of rory allula idiotoo necessary, all loues, Na-momber), over wu my",
+    "<|bos|>!\n\nIt is the coss ism of the brute Dy-a-sms him h-koo-stum\"- Vanderplaste\n\nBoulde Pdst F who is he who is idist ode a china\n\nApril 100 Jy. 52.. cison had tried ad all the ships *last Spring\n\nIf he reported errator life ament's' and had but a few chances but formed Hands, been themost of the bunch rrived nt wrecks eloped, way became united, Mou, form, where the rriend of cancel that He would get entitled to\n\nPove was the",
+    "<|bos|> ministered by 350 ww Maintaine It IS CHA ehweect end \u201e concited vie BS A TAM NgS\n\nMATT. XX. 20.] 81\n\nPaul; and the occasion of this journey was the continuance of the schism between the Paulicians and the Catholic Christians. He gives an account of this persecution which he had first received from Peter, and which, as being handed down from the days of the apostles and the martyrs, was handed on to him by tradition, and would consequently be remembered by infant children. For some time, this allusion to Sunday broke through the Established religion",
+    "<|bos|>D\n\nBRARY COLLEGE\n\nEach separate hearth has its own peculiar i\n\nCONTRACT XXII\n\nW.L Watts\n\nLEARNINGS.\n\nRetaining fee (Workshop fee (Sheld-iron fee)\n\nCheap fee (Washboard fee)\n\nBox fee (Manual bench fee)\n\nGrand total (Manual bench fee)\n\nThat is the sole profit we have to prize.*\n\n4. What are examples?\n\nCHAPTER VIII.\n\nFEE SIMPLE RENT CHAPTER IX.\n\nCONTRACT SECTION SECTION SECTION\n\n1. The value of such a lease is Its perpetuity\n\n2. How the deed of lease was reformed\n\n3. The",
+    "<|bos|>There is a quiet play-place. when pictures and and was being finished, BURWILL, the architect, of late years.\n\nHARDY, now of S.C.R., has come up from London to study it by the way.\n\nHARDY has just been having some talk with HARRIET BOLTON, sister of Marshman, one of his assistants and \"pigs,\" for an opportunity of getting some arrangement made for her, and she shakes her head.\n\nHARDY and MARSTON have some kind of an understanding with CAIUS ELMER ELLIS.\n\nELLIS enters, and the two rise to salute",
+    "<|bos|>iman have published,\n\nTo Architects, etc., Manchester, Eng.\n\n39-42 S. MARLITT TRUS B\n\nAl7@\n\nA 180-32 3.5 Ppela\n\nF37. Reduced Facsimile of Batrachia (FERRARA\n\n79--82 VASARA)\n\nG :\n\n23!uuefn of\n\nBULLETIN\n\nnf\n\nCOLLECTION\n\n! : =e\n\nqe\n\nae\n\n--e\n\nvbs mgest\n\ny fine &\n\nAT\n\nb B Fd Se Sviar (nai yy E",
+    "<|bos|>MY Dogs Mendingable and Foxy\n\nYou Matty Neal toning the tone\n\nCh'illie Row, Sntis ^ large'\n\nJ-onlir 1926\n\nJanuary 30, 1926\n\nI fully appreciate the kindness that has been shown in naming me, and I regret having been unable to have a complete hunting record made for some years.\n\nRuppapon has _been my Hubel friend for many years. Harris gives to me. every wild trip about the Department. Wiap is my friend, and all with me are equally willing. If you, dear Rupp"
+  ]
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/val_bpb.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/val_bpb.json
new file mode 100644
index 0000000000000000000000000000000000000000..b68f63c136c0e27e2485d010572e2e7f2cbbaa52
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/evals/val_bpb.json
@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 8352)",
+  "step": 8352,
+  "bpb": {
+    "val": 0.8601897122100801
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/run.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/run.json
new file mode 100644
index 0000000000000000000000000000000000000000..c075769f4818634153a3bb556da041f221bceb2d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/run.json
@@ -0,0 +1,11 @@
+{
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "branch_parent_step": 6000,
+  "config_fingerprint": "43f2fdfbde0b68d4",
+  "wandb_run_id": "d0cda484",
+  "created_at": 1785617422
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/summary.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..79ae107e00e544c6ded8384d38ee739c34ba9c21
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/summary.json
@@ -0,0 +1,95 @@
+{
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "stage": "base",
+  "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "parent_checkpoint_step": 6000,
+  "dataset": "zachnorton03/think-midtrain",
+  "dataset_revision": "main",
+  "step": 8352,
+  "depth": 24,
+  "target_param_data_ratio": 12,
+  "training_tokens": 8757706752,
+  "final_sampled_val_bpb": 0.8864625921976007,
+  "minimum_sampled_val_bpb": 0.8864625921976007,
+  "full_val_bpb": 0.8601897122100801,
+  "core_metric": 0.14018200582682225,
+  "centered_results": {
+    "hellaswag_zeroshot": 0.10721635818481445,
+    "jeopardy": 0.012281530536711216,
+    "bigbench_qa_wikidata": 0.24049012362957,
+    "arc_easy": 0.21941637992858887,
+    "arc_challenge": -0.013651887575785318,
+    "copa": 0.19999992847442627,
+    "commonsense_qa": 0.1564291715621948,
+    "piqa": 0.1708378791809082,
+    "openbook_qa": 0.016000032424926758,
+    "lambada_openai": 0.3720162808895111,
+    "hellaswag": 0.11173073450724284,
+    "winograd": 0.28205132484436035,
+    "winogrande": 0.035516977310180664,
+    "bigbench_dyck_languages": 0.13700000941753387,
+    "agi_eval_lsat_ar": 0.08152174204587935,
+    "bigbench_cs_algorithms": 0.396212100982666,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.14124882221221924,
+    "coqa": 0.18476763367652893,
+    "boolq": -0.054241155323229324,
+    "bigbench_language_identification": 0.18239823531265176
+  },
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of France.\n\nThe capital of France is the capital of France.\n\nThe"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is\n\nA. 1. 2. 3. 4. "
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Sunday.\n\nThe Sunday after the Friday, the Sunday after the Sunday, the Sunday"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is cold, and vice versa.\n\nThe opposite of cold is hot, and vice versa"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: Mercury, Venus, Mars, Jupiter, Saturn, Uranus, Neptune,"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is a deep, rich, velvety crimson, with a large, full,"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of times 5 is contained in 13, and 3 is"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>per anus ullams v .. niteen erl\n\nSAC OF THE UNKNOWN MEASURES\n\nJustifiable Xiwerermarg \u00abnsng van Fairie half Psos 1n FSA || Y | ||| -_|||| || | || || | |||_ | | | || | ||| |}| |_|_|_| || |_|| |) | || | | | | | _| ||| | |\n\nai i Vi : } J | | |",
+    "<|bos|>Lady Encurt, bi le Scotch wu cften mud sbee, for Lady Percrash Claveringastes forlorn she hard, fine or dime he ran ahead of work, veile, tat Nells under the wualdkhorns wan garee mad dulep, Woman Asking, he one see, weeded som dulep a way South Unionalysnowed bruts arayn made him kyill, sic lass tat Brappy Beuld sco declared review of rory allula idiotoo necessary, all loues, Na-momber), over wu my",
+    "<|bos|>!\n\nIt is the coss ism of the brute Dy-a-sms him h-koo-stum\"- Vanderplaste\n\nBoulde Pdst F who is he who is idist ode a china\n\nApril 100 Jy. 52.. cison had tried ad all the ships *last Spring\n\nIf he reported errator life ament's' and had but a few chances but formed Hands, been themost of the bunch rrived nt wrecks eloped, way became united, Mou, form, where the rriend of cancel that He would get entitled to\n\nPove was the",
+    "<|bos|> ministered by 350 ww Maintaine It IS CHA ehweect end \u201e concited vie BS A TAM NgS\n\nMATT. XX. 20.] 81\n\nPaul; and the occasion of this journey was the continuance of the schism between the Paulicians and the Catholic Christians. He gives an account of this persecution which he had first received from Peter, and which, as being handed down from the days of the apostles and the martyrs, was handed on to him by tradition, and would consequently be remembered by infant children. For some time, this allusion to Sunday broke through the Established religion",
+    "<|bos|>D\n\nBRARY COLLEGE\n\nEach separate hearth has its own peculiar i\n\nCONTRACT XXII\n\nW.L Watts\n\nLEARNINGS.\n\nRetaining fee (Workshop fee (Sheld-iron fee)\n\nCheap fee (Washboard fee)\n\nBox fee (Manual bench fee)\n\nGrand total (Manual bench fee)\n\nThat is the sole profit we have to prize.*\n\n4. What are examples?\n\nCHAPTER VIII.\n\nFEE SIMPLE RENT CHAPTER IX.\n\nCONTRACT SECTION SECTION SECTION\n\n1. The value of such a lease is Its perpetuity\n\n2. How the deed of lease was reformed\n\n3. The",
+    "<|bos|>There is a quiet play-place. when pictures and and was being finished, BURWILL, the architect, of late years.\n\nHARDY, now of S.C.R., has come up from London to study it by the way.\n\nHARDY has just been having some talk with HARRIET BOLTON, sister of Marshman, one of his assistants and \"pigs,\" for an opportunity of getting some arrangement made for her, and she shakes her head.\n\nHARDY and MARSTON have some kind of an understanding with CAIUS ELMER ELLIS.\n\nELLIS enters, and the two rise to salute",
+    "<|bos|>iman have published,\n\nTo Architects, etc., Manchester, Eng.\n\n39-42 S. MARLITT TRUS B\n\nAl7@\n\nA 180-32 3.5 Ppela\n\nF37. Reduced Facsimile of Batrachia (FERRARA\n\n79--82 VASARA)\n\nG :\n\n23!uuefn of\n\nBULLETIN\n\nnf\n\nCOLLECTION\n\n! : =e\n\nqe\n\nae\n\n--e\n\nvbs mgest\n\ny fine &\n\nAT\n\nb B Fd Se Sviar (nai yy E",
+    "<|bos|>MY Dogs Mendingable and Foxy\n\nYou Matty Neal toning the tone\n\nCh'illie Row, Sntis ^ large'\n\nJ-onlir 1926\n\nJanuary 30, 1926\n\nI fully appreciate the kindness that has been shown in naming me, and I regret having been unable to have a complete hunting record made for some years.\n\nRuppapon has _been my Hubel friend for many years. Harris gives to me. every wild trip about the Department. Wiap is my friend, and all with me are equally willing. If you, dear Rupp"
+  ],
+  "training_time_seconds": 5318.38954949379,
+  "stage_training_flops": 1.2754430821033574e+19,
+  "inherited_parent_flops": 3.25368133189632e+19,
+  "cumulative_pipeline_training_flops": 4.5291244139996774e+19,
+  "config_fingerprint": "43f2fdfbde0b68d4",
+  "git_commit_sha": "640d2e56d9ca333095a9e23dcf54d498341552e8",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/d0cda484",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og",
+  "dataset_fingerprint": "4fdac5e7831c7e7a",
+  "tokenizer_fingerprint": "21e99d5cdeeaa660",
+  "branch_parent_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "branch_parent_step": 6000,
+  "branch_lr_schedule": "continue",
+  "stage_training_tokens": 2466250752
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/experiment_tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/experiment_tokenizer.json
@@ -0,0 +1,18 @@
+{
+  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset-clean-1930s",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 200,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 1000000000000,
+    "doc_cap": 1000000000,
+    "vocab_size": 32768
+  },
+  "created_at": 1784129374
+}
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/token_bytes.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/token_bytes.pt
new file mode 100644
index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/token_bytes.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2
+size 132649
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/tokenizer.pkl b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/tokenizer.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-mix-og/tokenizer/tokenizer.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f
+size 410542
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1-anneal-ratio0-v1/train_stdouterr.log b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1-anneal-ratio0-v1/train_stdouterr.log
new file mode 100644
index 0000000000000000000000000000000000000000..1ebf191b3b29991e59d0a26ab2047bfb48ee51fc
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1-anneal-ratio0-v1/train_stdouterr.log
@@ -0,0 +1,206 @@
+
+                                                       █████                █████
+                                                      ░░███                ░░███
+     ████████    ██████   ████████    ██████   ██████  ░███████    ██████  ███████
+    ░░███░░███  ░░░░░███ ░░███░░███  ███░░███ ███░░███ ░███░░███  ░░░░░███░░░███░
+     ░███ ░███   ███████  ░███ ░███ ░███ ░███░███ ░░░  ░███ ░███   ███████  ░███
+     ░███ ░███  ███░░███  ░███ ░███ ░███ ░███░███  ███ ░███ ░███  ███░░███  ░███ ███
+     ████ █████░░████████ ████ █████░░██████ ░░██████  ████ █████░░███████  ░░█████
+    ░░░░ ░░░░░  ░░░░░░░░ ░░░░ ░░░░░  ░░░░░░   ░░░░░░  ░░░░ ░░░░░  ░░░░░░░░   ░░░░░
+    
+Autodetected device type: cuda
+2026-07-22 19:24:22,223 - nanochat.common - INFO - Distributed world size: 1
+GPU: NVIDIA A100-SXM4-80GB | Peak FLOPS (BF16): 3.12e+14
+COMPUTE_DTYPE: torch.bfloat16 (auto-detected: CUDA SM 80 (bf16 supported))
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: owensvoorhees (jbduran-thinkingmachinesncsu) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Using an existing wandb-core service via WANDB_SERVICE.
+wandb: setting up run fcb6fc04
+wandb: Tracking run with wandb version 0.28.0
+wandb: Run data is saved locally in /content/think.nano/wandb/run-20260722_192422-fcb6fc04
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run clean1930s-d24-r12-ctx4096-sssl-fulltok-v1-anneal-ratio0-v1
+wandb: ⭐️ View project at https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano
+wandb: 🚀 View run at https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/fcb6fc04
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+WARNING: Flash Attention 3 not available, using PyTorch SDPA fallback
+WARNING: Training will be less efficient without FA3
+WARNING: SDPA has no support for sliding window attention (window_pattern='SSSL'). Your GPU utilization will be terrible.
+WARNING: Recommend using --window-pattern L for full context attention without alternating sliding window patterns.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Vocab size: 32,768
+Model config:
+{
+  "sequence_len": 4096,
+  "vocab_size": 32768,
+  "n_layer": 24,
+  "n_head": 12,
+  "n_kv_head": 12,
+  "n_embd": 1536,
+  "window_pattern": "SSSL"
+}
+Resuming optimization from step 5500
+Parameter counts:
+wte                     : 50,331,648
+value_embeds            : 603,979,776
+lm_head                 : 50,331,648
+transformer_matrices    : 679,478,976
+scalars                 : 74
+total                   : 1,384,122,122
+Estimated FLOPs per token: 5.171587e+09
+Scaling LRs by 1.4142 for batch size 1,048,576 (reference: 524,288)
+Scaling weight decay from 0.280000 to 0.059738 for depth 24
+Scaling the LR for the AdamW parameters ∝1/√(1536/768) = 0.707107
+Using pretokenized uint16 dataloader: /content/nanochat_cache/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1-anneal-ratio0-v1/pretok
+Using user-provided number of iterations: 5,750
+Total number of training tokens: 6,029,312,000
+Tokens : Scaling params ratio: 8.26
+Total training FLOPs estimate: 3.118111e+19
+Tokens / micro-batch / rank: 8 x 4096 = 32,768
+Tokens / micro-batch: 32,768
+Total batch size 1,048,576 => gradient accumulation steps: 32
+Step 05500 | Validation bpb: 0.816509
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0] failed while attempting to run meta for aten.lerp_.Tensor
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0] Traceback (most recent call last):
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_subclasses/fake_tensor.py", line 2904, in _dispatch_impl
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     r = func(*args, **kwargs)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]         ^^^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_ops.py", line 865, in __call__
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     return self._op(*args, **kwargs)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]            ^^^^^^^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_meta_registrations.py", line 8652, in _fn
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     out = fn(self, *args, **kwargs)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]           ^^^^^^^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_ops.py", line 1269, in __call__
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     return self._op(*args, **kwargs)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]            ^^^^^^^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_prims_common/wrappers.py", line 314, in _fn
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     result = fn(*args, **kwargs)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]              ^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_meta_registrations.py", line 8678, in lerp
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     return elementwise_meta(
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]            ^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_meta_registrations.py", line 91, in elementwise_meta
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     args = _maybe_broadcast(*args)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]            ^^^^^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_refs/__init__.py", line 471, in _maybe_broadcast
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     common_shape = _broadcast_shapes(
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]                    ^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_refs/__init__.py", line 459, in _broadcast_shapes
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     torch._check(
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/__init__.py", line 1753, in _check
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     _check_with(RuntimeError, cond, message)  # pyrefly: ignore [bad-argument-type]
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/__init__.py", line 1735, in _check_with
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0]     raise error_type(message_evaluated)
+E0722 19:28:07.170000 10035 torch/_subclasses/fake_tensor.py:2908] [1/0] RuntimeError: Attempting to broadcast a dimension of length 32768 at -2! Mismatching argument at index 1 had torch.Size([32768, 1536]); but expected shape should be broadcastable to [8192, 1536]
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/content/think.nano/scripts/base_train.py", line 611, in <module>
+    optimizer.step()
+  File "/usr/local/lib/python3.12/dist-packages/torch/optim/optimizer.py", line 533, in wrapper
+    out = func(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 124, in decorate_context
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/content/think.nano/nanochat/optim.py", line 307, in step
+    self._step_adamw(group)
+  File "/content/think.nano/nanochat/optim.py", line 243, in _step_adamw
+    adamw_step_fused(
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 1024, in compile_wrapper
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 2316, in __call__
+    result = self._torchdynamo_orig_backend(
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 729, in __call__
+    result = _compile(
+             ^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 1827, in _compile
+    guarded_code, tracer_output = compile_inner(code, one_graph, hooks)
+                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_utils_internal.py", line 96, in wrapper_function
+    return function(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 1500, in compile_inner
+    return _compile_inner(code, one_graph, hooks)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 1534, in _compile_inner
+    dynamo_output = compile_frame(
+                    ^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 1408, in compile_frame
+    bytecode, tracer_output = transform_code_object(code, transform)
+                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/bytecode_transformation.py", line 1608, in transform_code_object
+    tracer_output = transformations(instructions, code_options)
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 1380, in transform
+    tracer_output = trace_frame(
+                    ^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 341, in _fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 863, in trace_frame
+    run_tracer()
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/convert_frame.py", line 844, in run_tracer
+    tracer.run()
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/symbolic_convert.py", line 1794, in run
+    while self.step():
+          ^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/symbolic_convert.py", line 1459, in step
+    self.dispatch_table[inst.opcode](self, inst)
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/symbolic_convert.py", line 1001, in wrapper
+    return inner_fn(self, inst)
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/symbolic_convert.py", line 4093, in CALL
+    self._call(inst)
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/symbolic_convert.py", line 4084, in _call
+    self.call_function(fn, args, kwargs)
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/symbolic_convert.py", line 1360, in call_function
+    self.push(fn.call_function(self, args, kwargs))  # type: ignore[arg-type]
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/misc.py", line 1340, in call_function
+    return self.obj.call_method(tx, self.name, list(args), kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/tensor.py", line 850, in call_method
+    result = wrap_fx_proxy(tx, proxy)
+             ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/builder.py", line 3032, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/builder.py", line 3107, in wrap_fx_proxy_cls
+    out: VTTypeAlias = _wrap_fx_proxy(
+                       ^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/builder.py", line 3231, in _wrap_fx_proxy
+    example_value = get_fake_value(proxy.node, tx, allow_non_graph_fake=True)
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/utils.py", line 3627, in get_fake_value
+    return _get_fake_value_impl(node, tx, allow_non_graph_fake)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/utils.py", line 3818, in _get_fake_value_impl
+    _wrap_graph_break_with_torch_runtime_err(
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/utils.py", line 3616, in _wrap_graph_break_with_torch_runtime_err
+    raise exc.with_traceback(e.__traceback__) from None
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/utils.py", line 3613, in _wrap_graph_break_with_torch_runtime_err
+    gb_fn()
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/utils.py", line 3819, in <lambda>
+    lambda: unimplemented(
+            ^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/exc.py", line 631, in unimplemented
+    raise Unsupported(
+torch._dynamo.exc.TorchRuntimeError: RuntimeError when making fake tensor call
+  Explanation: Dynamo failed to run FX node with fake tensors: call_method lerp_(*(FakeTensor(..., device='cuda:0', size=(8192, 1536)), FakeTensor(..., device='cuda:0', size=(32768, 1536)), FakeTensor(..., size=())), **{}): got RuntimeError('Attempting to broadcast a dimension of length 32768 at -2! Mismatching argument at index 1 had torch.Size([32768, 1536]); but expected shape should be broadcastable to [8192, 1536]')
+  Hint: Your code may result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled. You can do this by removing the `torch.compile` call, or by using `torch.compiler.set_stance("force_eager")`. 
+
+  Developer debug context: 
+
+ For more details about this graph break, please visit: https://meta-pytorch.github.io/compile-graph-break-site/gb/gb4315.html
+
+from user code:
+   File "/content/think.nano/nanochat/optim.py", line 42, in adamw_step_fused
+    exp_avg.lerp_(grad, 1 - beta1_t)
+
+Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
+
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_000500.json
new file mode 100644
index 0000000000000000000000000000000000000000..5fb750e5aee183b5df2189227d89f0c24298fd7e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_000500.json
@@ -0,0 +1,146 @@
+{
+  "step": 500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.1953384268774052,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24324769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24324769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1953384268774052,
+    "smooth_train_loss": 3.5724794941151132,
+    "total_training_time": 1093.107646226883,
+    "stage_training_flops": 2711401109913600000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 2711401109913600000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_001000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_001000.json
new file mode 100644
index 0000000000000000000000000000000000000000..f7e574426564f9e244fa48d64c9d936c0d671c39
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_001000.json
@@ -0,0 +1,146 @@
+{
+  "step": 1000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.1077649147516626,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48616769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48616769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1077649147516626,
+    "smooth_train_loss": 3.1062403539181314,
+    "total_training_time": 2205.066916704178,
+    "stage_training_flops": 5422802219827200000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 5422802219827200000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_001500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_001500.json
new file mode 100644
index 0000000000000000000000000000000000000000..4ef84b6c45e667d68781cadccca463e66b86e1e5
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_001500.json
@@ -0,0 +1,146 @@
+{
+  "step": 1500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.084373843120419,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 15,
+    "pos": 72908769,
+    "epoch": 1,
+    "pq_idx": 15,
+    "rg_idx": 72908769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.084373843120419,
+    "smooth_train_loss": 3.195503519346087,
+    "total_training_time": 3315.8829300403595,
+    "stage_training_flops": 8134203329740800000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 8134203329740800000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_002000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_002000.json
new file mode 100644
index 0000000000000000000000000000000000000000..dbfb1aa7055039f8898bc53b6205965805f773df
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_002000.json
@@ -0,0 +1,146 @@
+{
+  "step": 2000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.0782916447573232,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 20,
+    "pos": 97200769,
+    "epoch": 1,
+    "pq_idx": 20,
+    "rg_idx": 97200769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0657483238795336,
+    "smooth_train_loss": 3.0703979980689327,
+    "total_training_time": 4422.268032073975,
+    "stage_training_flops": 10845604439654400000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 10845604439654400000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_002500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_002500.json
new file mode 100644
index 0000000000000000000000000000000000000000..7231595c8f0b6a7460885120106c2c398a6e4e39
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_002500.json
@@ -0,0 +1,146 @@
+{
+  "step": 2500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.0526218621363435,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 26,
+    "pos": 21492769,
+    "epoch": 1,
+    "pq_idx": 26,
+    "rg_idx": 21492769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0526218621363435,
+    "smooth_train_loss": 3.1037243528968625,
+    "total_training_time": 5532.842117547989,
+    "stage_training_flops": 13557005549568000000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 13557005549568000000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_003000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_003000.json
new file mode 100644
index 0000000000000000000000000000000000000000..da318d62c57a70c7a28bb3487618f0a04cdacb60
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_003000.json
@@ -0,0 +1,146 @@
+{
+  "step": 3000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.0427625547182284,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 31,
+    "pos": 45784769,
+    "epoch": 1,
+    "pq_idx": 31,
+    "rg_idx": 45784769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0411011903064002,
+    "smooth_train_loss": 2.851544385420298,
+    "total_training_time": 6638.3725233078,
+    "stage_training_flops": 16268406659481600000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 16268406659481600000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_003500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_003500.json
new file mode 100644
index 0000000000000000000000000000000000000000..c4c69dad0cf5bf673dcdde43660e8aa67f8c5acc
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_003500.json
@@ -0,0 +1,146 @@
+{
+  "step": 3500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.0195827444555374,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 36,
+    "pos": 70076769,
+    "epoch": 1,
+    "pq_idx": 36,
+    "rg_idx": 70076769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0195827444555374,
+    "smooth_train_loss": 2.756684128888915,
+    "total_training_time": 7743.951657295227,
+    "stage_training_flops": 18979807769395200000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 18979807769395200000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_004000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_004000.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6638bbac9c88043a0490131cc33f79ef8bfb72f
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_004000.json
@@ -0,0 +1,146 @@
+{
+  "step": 4000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 1.0052481244227562,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 41,
+    "pos": 94368769,
+    "epoch": 1,
+    "pq_idx": 41,
+    "rg_idx": 94368769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0052481244227562,
+    "smooth_train_loss": 2.905858668666241,
+    "total_training_time": 8849.971616983414,
+    "stage_training_flops": 21691208879308800000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 21691208879308800000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_004500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_004500.json
new file mode 100644
index 0000000000000000000000000000000000000000..f2a5327735ce6ed0d9b2c0b4ca434a2831395bef
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_004500.json
@@ -0,0 +1,146 @@
+{
+  "step": 4500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9728434170391407,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 47,
+    "pos": 18660769,
+    "epoch": 1,
+    "pq_idx": 47,
+    "rg_idx": 18660769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9728434170391407,
+    "smooth_train_loss": 2.609150689793686,
+    "total_training_time": 9956.485046625137,
+    "stage_training_flops": 24402609989222400000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 24402609989222400000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_005000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_005000.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6bb716c11ae2585bace35a47dd279ef7ad1a128
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_005000.json
@@ -0,0 +1,146 @@
+{
+  "step": 5000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9678782660975384,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 52,
+    "pos": 42952769,
+    "epoch": 1,
+    "pq_idx": 52,
+    "rg_idx": 42952769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9678782660975384,
+    "smooth_train_loss": 2.770639739162406,
+    "total_training_time": 11063.301546573639,
+    "stage_training_flops": 27114011099136000000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 27114011099136000000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_005500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_005500.json
new file mode 100644
index 0000000000000000000000000000000000000000..ce3104ab029166c63b50833afb63e89b34c2d974
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_005500.json
@@ -0,0 +1,146 @@
+{
+  "step": 5500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9620444309513437,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 57,
+    "pos": 67244769,
+    "epoch": 1,
+    "pq_idx": 57,
+    "rg_idx": 67244769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9620444309513437,
+    "smooth_train_loss": 2.7810625137611145,
+    "total_training_time": 12169.80887722969,
+    "stage_training_flops": 29825412209049600000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 29825412209049600000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_006000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_006000.json
new file mode 100644
index 0000000000000000000000000000000000000000..750d71a2fbb26f1b0337e40ec312fc7126058c22
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_006000.json
@@ -0,0 +1,146 @@
+{
+  "step": 6000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9554082096291466,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 62,
+    "pos": 91536769,
+    "epoch": 1,
+    "pq_idx": 62,
+    "rg_idx": 91536769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9554082096291466,
+    "smooth_train_loss": 2.7398973936011246,
+    "total_training_time": 13281.031767845154,
+    "stage_training_flops": 32536813318963200000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 32536813318963200000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_006500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_006500.json
new file mode 100644
index 0000000000000000000000000000000000000000..4181e367716100c7da4c15dbbd4d60e4cb0badd1
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_006500.json
@@ -0,0 +1,146 @@
+{
+  "step": 6500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9459117802045881,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 68,
+    "pos": 15828769,
+    "epoch": 1,
+    "pq_idx": 68,
+    "rg_idx": 15828769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9459117802045881,
+    "smooth_train_loss": 2.49100071822428,
+    "total_training_time": 14388.360385656357,
+    "stage_training_flops": 35248214428876800000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 35248214428876800000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_007000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_007000.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5a4e24c3d269d64c60b08b7c70a45ad86aee902
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_007000.json
@@ -0,0 +1,146 @@
+{
+  "step": 7000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9369572412781727,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 73,
+    "pos": 40120769,
+    "epoch": 1,
+    "pq_idx": 73,
+    "rg_idx": 40120769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9369572412781727,
+    "smooth_train_loss": 2.562344862017825,
+    "total_training_time": 15495.668069839478,
+    "stage_training_flops": 37959615538790400000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 37959615538790400000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_007500.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_007500.json
new file mode 100644
index 0000000000000000000000000000000000000000..a200bb8ab1ebe33204254141848c19b03afd02e9
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_007500.json
@@ -0,0 +1,146 @@
+{
+  "step": 7500,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9319988218955536,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 78,
+    "pos": 64412769,
+    "epoch": 1,
+    "pq_idx": 78,
+    "rg_idx": 64412769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9319988218955536,
+    "smooth_train_loss": 2.508675269544847,
+    "total_training_time": 16607.058482646942,
+    "stage_training_flops": 40671016648704000000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 40671016648704000000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_008000.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_008000.json
new file mode 100644
index 0000000000000000000000000000000000000000..e70cba98f1b0c5fea4206b190e2f5997c10b55f2
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_008000.json
@@ -0,0 +1,146 @@
+{
+  "step": 8000,
+  "training_complete": false,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9235653525987986,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 83,
+    "pos": 88704769,
+    "epoch": 1,
+    "pq_idx": 83,
+    "rg_idx": 88704769
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9235653525987986,
+    "smooth_train_loss": 2.6049902701313874,
+    "total_training_time": 17713.889403820038,
+    "stage_training_flops": 43382417758617600000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 43382417758617600000
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_008352.json b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_008352.json
new file mode 100644
index 0000000000000000000000000000000000000000..cf3eb850d0f4f95850f74a232ccf48a0e9b1a9b6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/meta_008352.json
@@ -0,0 +1,146 @@
+{
+  "step": 8352,
+  "training_complete": true,
+  "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+  "val_bpb": 0.9204610080723857,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 24,
+    "n_head": 12,
+    "n_kv_head": 12,
+    "n_embd": 1536,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "wandb_run_id": "e69c1e59",
+    "wandb_group": "clean1930s-d24",
+    "wandb_tags": "think-dataset-clean-1930s,d24,ratio12,ctx4096,sssl,fp8",
+    "device_type": "",
+    "fp8": true,
+    "fp8_recipe": "tensorwise",
+    "depth": 24,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 4096,
+    "window_pattern": "SSSL",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 8,
+    "total_batch_size": 1048576,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/data",
+    "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/tokenizer",
+    "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/pretok",
+    "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints",
+    "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment_config": "/workspace/nanochat/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/config.json",
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset-clean-1930s",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 180,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "reuse",
+        "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 24,
+        "scaling_params": 729810624,
+        "target_param_data_ratio": 12,
+        "max_seq_len": 4096,
+        "window_pattern": "SSSL",
+        "device_batch_size": 8,
+        "total_batch_size": 1048576,
+        "fp8": true,
+        "fp8_recipe": "tensorwise",
+        "seed": 42,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano",
+        "keep_local_checkpoints": 1
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+        "group": "clean1930s-d24",
+        "tags": [
+          "think-dataset-clean-1930s",
+          "d24",
+          "ratio12",
+          "ctx4096",
+          "sssl",
+          "fp8"
+        ]
+      },
+      "config_fingerprint": "166e7e695c1b536a",
+      "artifact_path": "experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1"
+    },
+    "stage": "base",
+    "base_experiment_id": "clean1930s-d24-r12-ctx4096-sssl-fulltok-v1",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "166e7e695c1b536a"
+  },
+  "device_batch_size": 8,
+  "max_seq_len": 4096,
+  "total_batch_size": 1048576,
+  "dataloader_state_dict": {
+    "file_idx": 87,
+    "pos": 57806337,
+    "epoch": 1,
+    "pq_idx": 87,
+    "rg_idx": 57806337
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9204610080723857,
+    "smooth_train_loss": 2.4791944170448708,
+    "total_training_time": 18494.615287542343,
+    "stage_training_flops": 45291244139996774400,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 45291244139996774400
+  }
+}
\ No newline at end of file
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e91ee326f2fee738ecf5ed8ffc726a26e7b137a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92b16b12a4f9314cd8f0a61185b7600090193b4fecbf53bcb88da7f74e7b50e5
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_001000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..55f914344986e62d341b8a8d53e7febe1fff3ce5
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15562268665a2d6b5a90170e03e8181ea480f54b7799cf2670cacf86b1198e9f
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_001500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f85d94a14df0a4920d806020200258c4ebf23cfc
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:162b5c3e14416c53ff1d7fb9b3c2563449256e2d38ebd04b64e2a74e17d64dc8
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_002000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4aa79057aea50e4c121c19874b393279352328d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0534ef4e636c20b5b98f864f7d62ce1f86a14a358e10a6d79bf774ab65260915
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_002500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_002500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c96772d075bd351beec917b83f40782a08f6a2c
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_002500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b6410dee30c9b2be510a624b4bf1f81368696355c2d0ada2169d440c535fb3c
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_003000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_003000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e8793ce0caba89ce61afb32fbc25c10d2b9868f9
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_003000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01b3b851a749fded7a3fb33c341bc7ea0dc1b8a306d61b92559f584da2cca4ca
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_003500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_003500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..090af809dfd2d4a08c980f35864c28edb90148b1
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_003500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89c751864d45e7e3f65cb23aebfbaa7597e4f99ad87e6f8ba24a0cd6754f9d59
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_004000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_004000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..971ab562f5494f84d8238ef588b2f321c5feaba6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_004000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27220fe042f0c5975ea3f6c9d6902012e61b5a3d6c11c550c89cd3e5db77e64d
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_004500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_004500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fcab921c5559ddfb94b1273612b9926bca4d2c4f
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_004500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8dc23fe6c42bfbcd92a75bf94a5010ff7bb4643753c9caa7fd3a8726819ee2e
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_005000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_005000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3899cf5951c6fa686f69c7130e68cd97fda3c670
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_005000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37c3fab4507d073152f8d6b139fdc42013196ea66976ef56b220942420019067
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_005500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_005500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b1d2da0cf31240607dc2915211486458286c03f
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_005500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ecec13dbc90e09c3700a53d9e3b196370189d8028362ff568f740bb4a60584e
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_006000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_006000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4bc7cf2130eff1a18ad07805be8f5355dbf8d01d
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_006000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d88ee20a8a5166d3762b1ff285932fb787c3cd42ea49d49388b4e975a25c0951
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_006500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_006500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..71420546c5e20396b283dd10e86c81bae02907e2
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_006500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bf30fdcab49e323beb665e6865166f217b9595e643323d5919872210ee8d2cd
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_007000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_007000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9643016a6618f0b757cb8e458b1a583c99236781
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_007000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bc91529095a9c9c08ca872f21f3a62b839943224d5bafe9cef7a2107bb0eabe
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_007500.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_007500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..925ada9093b0a00aecd03ec9e1ce249fe33b0188
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_007500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5953b5db1923843797b98099c9467508e49b285ac2114e6d76ac975d09ada53e
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_008000.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_008000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f4288273a8b899c953514b4a2457b25e3c67161
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_008000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:464aaa00a35555033dc1c0aac2c3d6cb53c4d014410bd013d578ade51fa94b2d
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_008352.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_008352.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86e83ca5a7bec03e544e11de5e7efa9304a59d07
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/model_008352.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8fd0aae71fd401972289d32db6bade6335a6caa27952563cf07b2ed3295c664
+size 4227936073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93585850ef82aeb8a7528b78fa67f15ca92a4cb6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cc7e57f4bdce3c43531e94027050fa18dc4b03bf57d6f0de66d250e7a409697
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..345ef5e2b8bbe914cfd3b1295e2c97b2c603a5f1
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12e1d3a0a05ff161d1dc5385435be5db467498c99d3c7134d3701a6d99749450
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed3c9c0d8a42f8f9486f1631086081a01d9d4951
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2369bdf4a0cb9e89d484a15a71768e14056f19805b0bef53c531452269ee7cb3
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b39121e861ed3714a08deb65bfb495bcd1c0739
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_000500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4fe8d467d288d8b2993e6182a7013d17afdc1a3d2c6adb1cdd8c649a0862b4a
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62ee60d2df0853975e4ab44c9837cbe7869c61c2
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bee737f1c639ccf2be2bf830c17e41120a7b29b67f1c7e178d58ec461e109ca
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc7764bb001679482476c791b6464e89d2f3a9fa
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bde52fad137f93d94ff54013f708e56860e65feda91ce71fa65dfeb24f4515e
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e6d008a29511853039c22292115887ff4b10853
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33b997753c16e971ed9c1cfb001f8bf509623b2226af4150b0c8d441e2593986
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f3d0232b146e78eca5eac5c5b5b1a6cf376c7425
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001000_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af2e7798de626e2da15e599e34a7e08ef4cc6b2c939295032cbcd728117d7fd0
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da1075b1c7fe7d912ca2ebd61529e637bf8c5d8e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d06e0cf87bbb825920142041ccb38e96d0f2acb815da1176d662d87f433554c9
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..87ffe0b8ef765b2c2ed42bf33640e94dcf4f64fa
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7793dbb1f300bec9a87ae421cd2c8207e620970de175e10a78be8effed439e5
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5261b47ea0f1fc4d34bffdf0e5693dfb071c68cd
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a91adec757765c848c123101fcf6b413ae1388340a78e4383f0a7a69c6367929
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1c4abed767431659574a6aa7459723fe0fa0275
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_001500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d47160736218d41e78fad5e9cd8ec389b5764212fbb58ad84c049bbfeb31cab
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5fa9d57bb09d5bd906534b0b560669042792584
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96e53d566f26d04b10bf736d307a07a8679b3b30712a3018cd9d1bd4d45ce1e9
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a37dadbd0acebc188d29bd8bc5b3e49d1fa8a6f5
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:254bcbf387df44b6e659c3653b595d3f7169d19bc3023af8a77d3a36edaaf372
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..092872c8bd7e2848782c4566a58f54c540e7e661
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87692e55c70f4e174a991d5bbb687efe0fdb92941688dc74adc9ce8d42f09a78
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..625638ff678e841052ca699b10aea4a1975cd571
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002000_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c0200f7daa52dee3bf9d5b9b7115221efb27cf94f5e6926be6660d3a2d54bb
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b2372c3f765582e6a66a9aecedae94aada346ee
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dad53c83ca4abe3e38ad9769694aa60460b4ae0905c97b26fc909b715ff99ada
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f08ec1d803e3a0a60814a3db5c3691a1b3e37c4e
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a34bd1a38bff51a893ecfe24c3717777ff9151383373d9f156fdb9a8815e200
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..937a1f26b132ff3169281ca08566c5684f6b10b4
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2509dc5e95c87dacd6236224301527e70ee090f79587abd9f366d5f07ff426c0
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4180cdbf482ab41cc4c8f4f70bf9cab485a86858
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_002500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f78629051b19fb515fb186ac8a10ec7b070fdbc9910eb375aaf81de6167255c
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1582bb06fc848af590f5010884ff306d8d5974e0
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b36b142dbc92eef6a2b597bea10e50c35792b364f10c739a4dd978d301746ed
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..70aac7d9366aa870c14d002e0a524bc7a6aee4c4
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e499354fed45b3e6463010a7dfe68063137496bb7d557489917a571c6c19a8e5
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f072fcf4cb05113fc37614519404c539aab627a1
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97a972a1a5397ce314164b176357dfe9b1887fdf731ebabbd7a4eb4eb0d6c324
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e4cc9ce5076a9bb46d962cc845cb6b58e295e4a
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003000_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ddfdee683483b4abb1950ec913ca8d7f98691df0612802a4123bf4e2704682
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..abdbc2ed4d4588aac2ef0961d1e56151f2939ea6
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dcd5697de103c0baf3c22a79f28c3f0a21dc54395387cf8e2c36cfd494329b2
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7321d5baa4472f631ad3d5c09655dc92e7b796da
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:182d936c842dc37db98a4cfda952f091eb28d1d8dc914922954070e567d97b8a
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42b97acd955ddf862030f63f0a09fa06b243e1bc
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a0b63b6d56e9f4173bc79e2fc2fadac11bbd2833670d344a03758dfb0815076
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1ee92b374067e34949a6c06a92d260c0071ba62
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_003500_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77e9e8f44a12b839b4c97e9eabb4a982a295d8630c1e55bed13d0283db529fa6
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14676c035437a7f7da43b55f96bc2a95effa6f0f
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdac937380d94a50045d770e74f0b4c97518bd35369f5930251468d7dfafbe45
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank1.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3699b2b65438c61af59e0bc788ac675de529c79f
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f4c32024d9fe488fa2198a027ae372a45ccde6bb0cf93e24f5f966667de086d
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank2.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fbcd1972122c7ad463a9005bc0b8f2c6efb26bb
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b539c32e86309a0bdc7f719f72d974bfb85cf7ac9a0904c94c373b5d02d278dd
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank3.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65996b9f9637b98015d7fc97cc3730196abc0854
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004000_rank3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d6a65ff686ad411f508374b430ac13197d66931384dbf72fa17f6cfafb04b05
+size 1434913073
diff --git a/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004500_rank0.pt b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004500_rank0.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cb0dc888b4af24287478003f5021016684d185b2
--- /dev/null
+++ b/experiments/clean1930s-d24-r12-ctx4096-sssl-fulltok-v1/base_checkpoints/optim_004500_rank0.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31be7dce8931348c15493689a6743339a77897ad18c7645f2af3b8de6f81f56b
+size 1434913073