Import experiment archive from bart (batch 5)

experiments/think-d12-1ep-25sh-r11-wd42/run.json

@@ -0,0 +1,10 @@
+{
+  "experiment_id": "think-d12-1ep-25sh-r11-wd42",
+  "stage": "base",
+  "base_experiment_id": "think-d12-1ep-25sh-r11-wd42",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "47bfa49108766b7d",
+  "wandb_run_id": "4e526b5a",
+  "created_at": 1782495190
+}

experiments/think-d12-1ep-25sh-r11-wd42/summary.json

@@ -0,0 +1,32 @@
+{
+  "experiment_id": "think-d12-1ep-25sh-r11-wd42",
+  "stage": "base",
+  "base_experiment_id": "think-d12-1ep-25sh-r11-wd42",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "dataset": "jbduran/think-dataset",
+  "dataset_revision": "main",
+  "step": 2362,
+  "depth": 12,
+  "target_param_data_ratio": 11.25,
+  "training_tokens": 1238368256,
+  "final_sampled_val_bpb": 1.103670265641304,
+  "minimum_sampled_val_bpb": 1.103670265641304,
+  "full_val_bpb": 1.0526348691238439,
+  "core_metric": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": [],
+  "training_time_seconds": 6293.967695713043,
+  "stage_training_flops": 1.0985538644638433e+18,
+  "inherited_parent_flops": 0.0,
+  "cumulative_pipeline_training_flops": 1.0985538644638433e+18,
+  "config_fingerprint": "47bfa49108766b7d",
+  "git_commit_sha": "7e29503cca7b67e1418323c3628e6f935e4feacf",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/4e526b5a",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-1ep-25sh-r11-wd42",
+  "dataset_fingerprint": "a6e1b3a100e0d8b3",
+  "tokenizer_fingerprint": "ebb3705d7792a34d",
+  "unique_train_tokens": 1275519304,
+  "effective_epochs": 0.970873786164196
+}

experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/experiment_tokenizer.json

@@ -0,0 +1,18 @@
+{
+  "experiment_id": "think-d12-1ep-25sh-r11-wd42",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 44,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "created_at": 1782495209
+}

experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f928e04aa2ac37dd4064493240d1e73ecab7acb217c5a183311b0c523a3468
+size 132649

experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd583e3c35851c62295a1a0f6d688923e4f30649ac963b701ec2440fe8bc3e4f
+size 404221

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_000500.json

@@ -0,0 +1,141 @@
+{
+  "step": 500,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.3221519749129604,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.3221519749129604,
+    "smooth_train_loss": 3.7478778179789605,
+    "total_training_time": 1305.7518684864044,
+    "stage_training_flops": 232547388751872000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 232547388751872000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001000.json

@@ -0,0 +1,141 @@
+{
+  "step": 1000,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.2600323113203347,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2600323113203347,
+    "smooth_train_loss": 3.646794584039554,
+    "total_training_time": 2640.877459049225,
+    "stage_training_flops": 465094777503744000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 465094777503744000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001500.json

@@ -0,0 +1,141 @@
+{
+  "step": 1500,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.2377503150891866,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86488769,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86488769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2377503150891866,
+    "smooth_train_loss": 3.6301349812757526,
+    "total_training_time": 3977.6890711784363,
+    "stage_training_flops": 697642166255616000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 697642166255616000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002000.json

@@ -0,0 +1,141 @@
+{
+  "step": 2000,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.2014593386689274,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48640769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48640769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2014593386689274,
+    "smooth_train_loss": 3.4382217869051193,
+    "total_training_time": 5322.514421463013,
+    "stage_training_flops": 930189555007488000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 930189555007488000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002500.json

@@ -0,0 +1,141 @@
+{
+  "step": 2500,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.166274403385848,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 13,
+    "pos": 10792769,
+    "epoch": 1,
+    "pq_idx": 13,
+    "rg_idx": 10792769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.166274403385848,
+    "smooth_train_loss": 3.265478801787732,
+    "total_training_time": 6658.766751766205,
+    "stage_training_flops": 1162736943759360000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1162736943759360000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003000.json

@@ -0,0 +1,141 @@
+{
+  "step": 3000,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.1420328141160099,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 15,
+    "pos": 72944769,
+    "epoch": 1,
+    "pq_idx": 15,
+    "rg_idx": 72944769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1420328141160099,
+    "smooth_train_loss": 3.1094901625575497,
+    "total_training_time": 8003.383926391602,
+    "stage_training_flops": 1395284332511232000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1395284332511232000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003500.json

@@ -0,0 +1,141 @@
+{
+  "step": 3500,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.1108293726096388,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 18,
+    "pos": 35096769,
+    "epoch": 1,
+    "pq_idx": 18,
+    "rg_idx": 35096769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1108293726096388,
+    "smooth_train_loss": 3.0718457586789576,
+    "total_training_time": 9347.049030542374,
+    "stage_training_flops": 1627831721263104000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1627831721263104000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004000.json

@@ -0,0 +1,141 @@
+{
+  "step": 4000,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.0845218539469001,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 20,
+    "pos": 97248769,
+    "epoch": 1,
+    "pq_idx": 20,
+    "rg_idx": 97248769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0845218539469001,
+    "smooth_train_loss": 2.94610128781119,
+    "total_training_time": 10685.518072605133,
+    "stage_training_flops": 1860379110014976000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1860379110014976000
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004200.json

@@ -0,0 +1,141 @@
+{
+  "step": 4200,
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "val_bpb": 1.078128321044417,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-44sh-r20-wd42",
+    "wandb_run_id": "5c4fba8a",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.42,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints",
+    "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json",
+    "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+    "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-44sh-r20-wd42",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 44,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 20.0,
+        "window_pattern": "L",
+        "weight_decay": 0.42,
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-44sh-r20-wd42",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio20",
+          "44shards",
+          "1epoch",
+          "wd.42"
+        ]
+      },
+      "config_fingerprint": "a8d632854c2cd1bd",
+      "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "a8d632854c2cd1bd"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 22,
+    "pos": 2109569,
+    "epoch": 1,
+    "pq_idx": 22,
+    "rg_idx": 2109569
+  },
+  "loop_state": {
+    "min_val_bpb": 1.078128321044417,
+    "smooth_train_loss": 2.8475676426206853,
+    "total_training_time": 11219.814347743988,
+    "stage_training_flops": 1953398065515724800,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1953398065515724800
+  }
+}

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_000500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db546864ca01fbe6875d9c12a049bf0aa4232ba3a4dbdba03ecbdd1095a15e55
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5af8aecdefdbf085e32749ea53e8ce55622196b42c90634282458c34af4d5e1a
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7582f0335e8a47d987ffb85174e35b14fdb939ea858448b32e21c56d3384056
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8ead5070dd054e0ea8f29381274296936bedcbc712ffd4fd64b11adbda9e088
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77f5679e681a78bf061e9cf261a005a4c658b107e974be777f85e489d8ad1140
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ca8be32656940ecf969903f3b513a3e673c74e5e405322c584f57e85a13df0f
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b330320b63a95624ecc9ba05722dda9609593987c48a6ab90047c94c9f3d06f1
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7986875a99712f761a3f218dfefbca73547a1ae8b45f08ab9d95c7dec1b2a3f
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004200.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d641bed4b7bf0e787b469394a0c01c73d1d66655be72525a4c625e3f04c961a6
+size 792761690

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_000500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1525cb06d3f15e13db9115c5bb67d4ba9c751c6c2bc2a5901d615a785323bd3f
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1181d48ea007611efdd9c45275aa6cbc350d176bf736aef963b05953f4e2837f
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93fcab3d3b289e92644d578ebcc8b9ea9dd644fe3e36efd4ebda802e0f438b42
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6602307370d1412fd6f58d5e1b67da2edd22590385d684ec62d1640075409d99
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fde77b3022a5126a82a110fb379c4aec5775aa6440623ea42a6790c99a721422
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c081ca79e912393a33cf8e63d1795047a87d90735757f51df8ed38477ba851d5
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:328e0b85a4407f35d10e644574bad923147a25c24dfbc5e8ebf0b095c2424497
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:704c4809a117c46020fcc551cfef13bd6aa5259a74ce8ab36acfd9993b126436
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004200_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c8ff3203701457409f4a8b416d178fb66b3ced9764bda87bccf48e0c1990cb6
+size 1246165357

experiments/think-d12-1ep-44sh-r20-wd42/config.json

@@ -0,0 +1,59 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 44,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 12,
+    "scaling_params": 110100912,
+    "target_param_data_ratio": 20.0,
+    "window_pattern": "L",
+    "weight_decay": 0.42,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano"
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "think-d12-1ep-44sh-r20-wd42",
+    "group": "think-d12",
+    "tags": [
+      "think-dataset",
+      "d12",
+      "ratio20",
+      "44shards",
+      "1epoch",
+      "wd.42"
+    ]
+  },
+  "config_fingerprint": "a8d632854c2cd1bd",
+  "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42"
+}

experiments/think-d12-1ep-44sh-r20-wd42/evals/core.json

@@ -0,0 +1,56 @@
+{
+  "model": "base_model (step 4200)",
+  "step": 4200,
+  "bpb": {},
+  "core_metric": 0.07908813484502109,
+  "core_results": {
+    "hellaswag_zeroshot": 0.28251343965530396,
+    "jeopardy": 0.0009447330958209932,
+    "bigbench_qa_wikidata": 0.07814575731754303,
+    "arc_easy": 0.31607744097709656,
+    "arc_challenge": 0.2022184282541275,
+    "copa": 0.5699999928474426,
+    "commonsense_qa": 0.312039315700531,
+    "piqa": 0.5527747273445129,
+    "openbook_qa": 0.24800001084804535,
+    "lambada_openai": 0.26043081283569336,
+    "hellaswag": 0.2815176248550415,
+    "winograd": 0.5604395866394043,
+    "winogrande": 0.4980268180370331,
+    "bigbench_dyck_languages": 0.11500000208616257,
+    "agi_eval_lsat_ar": 0.260869562625885,
+    "bigbench_cs_algorithms": 0.4015151262283325,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.025922421365976334,
+    "coqa": 0.0899411216378212,
+    "boolq": 0.542201817035675,
+    "bigbench_language_identification": 0.25669997930526733
+  },
+  "centered_results": {
+    "hellaswag_zeroshot": 0.04335125287373861,
+    "jeopardy": 0.0009447330958209932,
+    "bigbench_qa_wikidata": 0.07814575731754303,
+    "arc_easy": 0.08810325463612874,
+    "arc_challenge": -0.06370876232783,
+    "copa": 0.13999998569488525,
+    "commonsense_qa": 0.14004914462566373,
+    "piqa": 0.10554945468902588,
+    "openbook_qa": -0.002666652202606201,
+    "lambada_openai": 0.26043081283569336,
+    "hellaswag": 0.04202349980672201,
+    "winograd": 0.1208791732788086,
+    "winogrande": -0.003946363925933838,
+    "bigbench_dyck_languages": 0.11500000208616257,
+    "agi_eval_lsat_ar": 0.07608695328235625,
+    "bigbench_cs_algorithms": 0.4015151262283325,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.025922421365976334,
+    "coqa": 0.0899411216378212,
+    "boolq": -0.20473206043243405,
+    "bigbench_language_identification": 0.1822882060563997
+  },
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-1ep-44sh-r20-wd42/evals/samples.json

@@ -0,0 +1,48 @@
+{
+  "model": "base_model (step 4200)",
+  "step": 4200,
+  "bpb": {},
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the world. The gold of the world is the"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be the last day of the week. \n\nI am, dear Sir, your most"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the best of all. \n\nThe best of all is the best of all."
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, which is the sun of the solar system. \n\n"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the same as that of the sun, and the same as that of the moon"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the first and second, and x is the number of the second"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>IT\n\nS. PERMISSION Dedicated to the CSTRORRS OF BOSTON. \n\nADVERTISEMENT \n\nWHILE I am following up the translation of the \"Saul,\" it will not be useless to point out to the different subscribers, and to each subscriber, the different phases, phases and phases of the phenomena presented - an appearance presented in all its phases and genera, and common to them all, so as to convey to all minds then present, an impression which is correct as it is true. \n\nYet a work is what it professes to be, and this, to a very considerable extent, ought to go on for ever",
+    "<|bos|>900 women, or 96,000 or \n\n105,000 inhabitants, and in general the Morningags. Their occupation was not pleasant. . Some of them were married, and had children. Of these the \n\nMorningags were the principal; the inferior were careful at school. \n\nTheir style is not unlike that of the Turkish potter. Here the grand aim is to obtain a common arena. . . . . To English readers of books, the present day is one of the most exciting and disappointing experiences in European literature. We carelessly allow this people to be the exclusive champions of civilization and true civilization.",
+    "<|bos|>NY besides that and never' 't can on' weighing ourselves. To be dead'st... . . or dreaming'st......\n\nOur'ret too comin' (2) to the ordinary, non.. \n\nFloors (1) to pleasoit person. \n\nSane (a) can amply and gratuitously as long-a.. \n\nOn' puntity, be more alive... \n\nWaiving, too secundity, let us (y) take... \n\nAre cases ofctority sufficient, sometimes, epis-ty, disELLY \n\nORKILL idea. \n\nPUR-BO",
+    "<|bos|>-boat built by John Eddy \n\nGreen... 623 \n\nJEFFERSON, JOHN (b. Jan. 1831), commenced business as hotel and tavern-keeper at \n\nFickenkamp, Cal., Nov. 26, 1827.. \n\n289; succeeded to business as hotel-keeper and proprietor Dec. 22, 1844.. \n\n320; commenced his business as hotel-keeper and thenceforth became a hotel and boarding-house keeper. \n\n323; successfully carried on business as hotel-keeper and thenceforth became a hotel and boarding-house keeper.. \n\n329; at end of ",
+    "<|bos|>. \n\nColored by Hugh Angola M'Nabbs, Commodore James E. Lightwood, Notables. This in- amidships. ventilation of the service contests in the testingroom of the national cemetery will facilitate the work among sailors who are anxious to see their fellow-patriots die.\n\nColored from a Painting, by Luella Vancouver, L.S. \n\nNoticed by Asa G-Giveno. Feather.\n\nThere are grave dangers to the hospital which must be avoided.\n\nAcres of Described by Charles. \n\nSir Francis Drake's Louisiana-Book, $ 1555-1571",
+    "<|bos|>. \n\n TRUSTEES.]. [The whole difference between the trust companies of his farm, Blodgett v. Nugent, 63 K. B. 521, and the trust companies of Reingeldt v. Wilbraham, 95 A. 118, 56 Am. St. Rep. 232, was merely a clear difference of intention.]\n\nIt was also clear there had been a necessary delivery of the sound overseas cart can Turnusey v. Metropolitan St., etc., R. Co., 58 L. R. A. 655, and was a clear and acquies",
+    "<|bos|>ING Cosmopolites. \n\nSurviving Evidence. \n\nAs to the main objects of the girl's introduction, not necessarily to a rendition of 820-44 22305 N229 Jones concludes that a report of 22-8 has been inserted in this case regarding]\n\nburying be given by me, and that it was thought that this would facilitate our future proceedings.\n\nSome attempt was made in vain to find the advertisement notificatione by defendant, but one defendant named as gruffly as the was, namely, the boy M\u00e1 vhdvr\u00e1 to groom of the horse;",
+    "<|bos|> gilt west, sundry small pieces of paper were found in his apartment.\n\n1762.] buoyant as lightning. This piece, preserved in a drawer in the library of the British House of Commons, was a very bad article, and not a few of the proprietor's horse fell off as it fell from him. His mother was crying, and a number of other women were taking care of their milk boxes under the bed-window. One of them stripped the stranger of his best clothes. After the fireman had been succeeded to some small articles which were every moment received with peculiar satisfaction by his buttons and clenched f"
+  ]
+}

experiments/think-d12-1ep-44sh-r20-wd42/evals/val_bpb.json

@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 4200)",
+  "step": 4200,
+  "bpb": {
+    "val": 1.0191548981297465
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-1ep-44sh-r20-wd42/run.json

@@ -0,0 +1,10 @@
+{
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "stage": "base",
+  "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "a8d632854c2cd1bd",
+  "wandb_run_id": "5c4fba8a",
+  "created_at": 1781881918
+}

experiments/think-d12-1ep-44sh-r20-wd42/summary.json

@@ -0,0 +1,93 @@
+{
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "stage": "base",
+  "base_experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "dataset": "jbduran/think-dataset",
+  "dataset_revision": "main",
+  "step": 4200,
+  "depth": 12,
+  "target_param_data_ratio": 20.0,
+  "training_tokens": 2202009600,
+  "final_sampled_val_bpb": 1.078128321044417,
+  "minimum_sampled_val_bpb": 1.078128321044417,
+  "full_val_bpb": 1.0191548981297465,
+  "core_metric": 0.07908813484502109,
+  "centered_results": {
+    "hellaswag_zeroshot": 0.04335125287373861,
+    "jeopardy": 0.0009447330958209932,
+    "bigbench_qa_wikidata": 0.07814575731754303,
+    "arc_easy": 0.08810325463612874,
+    "arc_challenge": -0.06370876232783,
+    "copa": 0.13999998569488525,
+    "commonsense_qa": 0.14004914462566373,
+    "piqa": 0.10554945468902588,
+    "openbook_qa": -0.002666652202606201,
+    "lambada_openai": 0.26043081283569336,
+    "hellaswag": 0.04202349980672201,
+    "winograd": 0.1208791732788086,
+    "winogrande": -0.003946363925933838,
+    "bigbench_dyck_languages": 0.11500000208616257,
+    "agi_eval_lsat_ar": 0.07608695328235625,
+    "bigbench_cs_algorithms": 0.4015151262283325,
+    "bigbench_operators": 0.10476190596818924,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.025922421365976334,
+    "coqa": 0.0899411216378212,
+    "boolq": -0.20473206043243405,
+    "bigbench_language_identification": 0.1822882060563997
+  },
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the world. The gold of the world is the"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be the last day of the week. \n\nI am, dear Sir, your most"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the best of all. \n\nThe best of all is the best of all."
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, which is the sun of the solar system. \n\n"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the same as that of the sun, and the same as that of the moon"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the first and second, and x is the number of the second"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>IT\n\nS. PERMISSION Dedicated to the CSTRORRS OF BOSTON. \n\nADVERTISEMENT \n\nWHILE I am following up the translation of the \"Saul,\" it will not be useless to point out to the different subscribers, and to each subscriber, the different phases, phases and phases of the phenomena presented - an appearance presented in all its phases and genera, and common to them all, so as to convey to all minds then present, an impression which is correct as it is true. \n\nYet a work is what it professes to be, and this, to a very considerable extent, ought to go on for ever",
+    "<|bos|>900 women, or 96,000 or \n\n105,000 inhabitants, and in general the Morningags. Their occupation was not pleasant. . Some of them were married, and had children. Of these the \n\nMorningags were the principal; the inferior were careful at school. \n\nTheir style is not unlike that of the Turkish potter. Here the grand aim is to obtain a common arena. . . . . To English readers of books, the present day is one of the most exciting and disappointing experiences in European literature. We carelessly allow this people to be the exclusive champions of civilization and true civilization.",
+    "<|bos|>NY besides that and never' 't can on' weighing ourselves. To be dead'st... . . or dreaming'st......\n\nOur'ret too comin' (2) to the ordinary, non.. \n\nFloors (1) to pleasoit person. \n\nSane (a) can amply and gratuitously as long-a.. \n\nOn' puntity, be more alive... \n\nWaiving, too secundity, let us (y) take... \n\nAre cases ofctority sufficient, sometimes, epis-ty, disELLY \n\nORKILL idea. \n\nPUR-BO",
+    "<|bos|>-boat built by John Eddy \n\nGreen... 623 \n\nJEFFERSON, JOHN (b. Jan. 1831), commenced business as hotel and tavern-keeper at \n\nFickenkamp, Cal., Nov. 26, 1827.. \n\n289; succeeded to business as hotel-keeper and proprietor Dec. 22, 1844.. \n\n320; commenced his business as hotel-keeper and thenceforth became a hotel and boarding-house keeper. \n\n323; successfully carried on business as hotel-keeper and thenceforth became a hotel and boarding-house keeper.. \n\n329; at end of ",
+    "<|bos|>. \n\nColored by Hugh Angola M'Nabbs, Commodore James E. Lightwood, Notables. This in- amidships. ventilation of the service contests in the testingroom of the national cemetery will facilitate the work among sailors who are anxious to see their fellow-patriots die.\n\nColored from a Painting, by Luella Vancouver, L.S. \n\nNoticed by Asa G-Giveno. Feather.\n\nThere are grave dangers to the hospital which must be avoided.\n\nAcres of Described by Charles. \n\nSir Francis Drake's Louisiana-Book, $ 1555-1571",
+    "<|bos|>. \n\n TRUSTEES.]. [The whole difference between the trust companies of his farm, Blodgett v. Nugent, 63 K. B. 521, and the trust companies of Reingeldt v. Wilbraham, 95 A. 118, 56 Am. St. Rep. 232, was merely a clear difference of intention.]\n\nIt was also clear there had been a necessary delivery of the sound overseas cart can Turnusey v. Metropolitan St., etc., R. Co., 58 L. R. A. 655, and was a clear and acquies",
+    "<|bos|>ING Cosmopolites. \n\nSurviving Evidence. \n\nAs to the main objects of the girl's introduction, not necessarily to a rendition of 820-44 22305 N229 Jones concludes that a report of 22-8 has been inserted in this case regarding]\n\nburying be given by me, and that it was thought that this would facilitate our future proceedings.\n\nSome attempt was made in vain to find the advertisement notificatione by defendant, but one defendant named as gruffly as the was, namely, the boy M\u00e1 vhdvr\u00e1 to groom of the horse;",
+    "<|bos|> gilt west, sundry small pieces of paper were found in his apartment.\n\n1762.] buoyant as lightning. This piece, preserved in a drawer in the library of the British House of Commons, was a very bad article, and not a few of the proprietor's horse fell off as it fell from him. His mother was crying, and a number of other women were taking care of their milk boxes under the bed-window. One of them stripped the stranger of his best clothes. After the fireman had been succeeded to some small articles which were every moment received with peculiar satisfaction by his buttons and clenched f"
+  ],
+  "training_time_seconds": 11219.814347743988,
+  "stage_training_flops": 1.9533980655157248e+18,
+  "inherited_parent_flops": 0.0,
+  "cumulative_pipeline_training_flops": 1.9533980655157248e+18,
+  "config_fingerprint": "a8d632854c2cd1bd",
+  "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/5c4fba8a",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-1ep-44sh-r20-wd42",
+  "dataset_fingerprint": "a6e1b3a100e0d8b3",
+  "tokenizer_fingerprint": "1744d7b7ee0d5d80",
+  "unique_train_tokens": 2268069888,
+  "effective_epochs": 0.970873786407767
+}

experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/experiment_tokenizer.json

@@ -0,0 +1,18 @@
+{
+  "experiment_id": "think-d12-1ep-44sh-r20-wd42",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 44,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "created_at": 1781881934
+}

experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f928e04aa2ac37dd4064493240d1e73ecab7acb217c5a183311b0c523a3468
+size 132649

experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd583e3c35851c62295a1a0f6d688923e4f30649ac963b701ec2440fe8bc3e4f
+size 404221

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_000500.json

@@ -0,0 +1,155 @@
+{
+  "step": 500,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.323730996039066,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.323730996039066,
+    "smooth_train_loss": 3.6702374931405064,
+    "total_training_time": 1313.3025135993958,
+    "stage_training_flops": 232547388751872000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 232547388751872000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001000.json

@@ -0,0 +1,155 @@
+{
+  "step": 1000,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.253497672143533,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.253497672143533,
+    "smooth_train_loss": 3.404809871021335,
+    "total_training_time": 2657.893961429596,
+    "stage_training_flops": 465094777503744000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 465094777503744000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001500.json

@@ -0,0 +1,155 @@
+{
+  "step": 1500,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.2322366946944483,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86488769,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86488769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2322366946944483,
+    "smooth_train_loss": 3.4686871369235353,
+    "total_training_time": 4000.771213531494,
+    "stage_training_flops": 697642166255616000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 697642166255616000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002000.json

@@ -0,0 +1,155 @@
+{
+  "step": 2000,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.212170813932778,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48640769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48640769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.212170813932778,
+    "smooth_train_loss": 3.525094410637873,
+    "total_training_time": 5345.096604824066,
+    "stage_training_flops": 930189555007488000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 930189555007488000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002500.json

@@ -0,0 +1,155 @@
+{
+  "step": 2500,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.1919329110762835,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 13,
+    "pos": 10792769,
+    "epoch": 1,
+    "pq_idx": 13,
+    "rg_idx": 10792769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1919329110762835,
+    "smooth_train_loss": 3.4785363277537242,
+    "total_training_time": 6687.278959035873,
+    "stage_training_flops": 1162736943759360000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1162736943759360000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003000.json

@@ -0,0 +1,155 @@
+{
+  "step": 3000,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.181723141516681,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 15,
+    "pos": 72944769,
+    "epoch": 1,
+    "pq_idx": 15,
+    "rg_idx": 72944769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.181723141516681,
+    "smooth_train_loss": 3.1625074370797437,
+    "total_training_time": 8029.638848543167,
+    "stage_training_flops": 1395284332511232000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1395284332511232000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003500.json

@@ -0,0 +1,155 @@
+{
+  "step": 3500,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.1594974101297872,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 18,
+    "pos": 35096769,
+    "epoch": 1,
+    "pq_idx": 18,
+    "rg_idx": 35096769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1594974101297872,
+    "smooth_train_loss": 3.1518777563692284,
+    "total_training_time": 9373.077644109726,
+    "stage_training_flops": 1627831721263104000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1627831721263104000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004000.json

@@ -0,0 +1,155 @@
+{
+  "step": 4000,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.1364783907083784,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 20,
+    "pos": 97248769,
+    "epoch": 1,
+    "pq_idx": 20,
+    "rg_idx": 97248769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1364783907083784,
+    "smooth_train_loss": 3.090057974445858,
+    "total_training_time": 10716.081592082977,
+    "stage_training_flops": 1860379110014976000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1860379110014976000
+  }
+}

experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004500.json

@@ -0,0 +1,155 @@
+{
+  "step": 4500,
+  "experiment_id": "think-d12-1ep-65sh-r30",
+  "val_bpb": 1.1270159041981165,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-1ep-65sh-r30",
+    "wandb_run_id": "6465e19b",
+    "wandb_group": "think-d12-stopping-point",
+    "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point",
+    "device_type": "cuda",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 30.0,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints",
+    "experiment_id": "think-d12-1ep-65sh-r30",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json",
+    "tokenizer_fingerprint": "db3bec0946e70097",
+    "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc",
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": 500,
+    "core_metric_max_per_task": 50,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-1ep-65sh-r30",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-1ep-65sh-r30",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 65,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "target_tokens": 3402104832,
+        "slack": 1.03,
+        "require_no_wrap": true,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "device_type": "cuda",
+        "depth": 12,
+        "aspect_ratio": 64,
+        "head_dim": 128,
+        "max_seq_len": 2048,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 30.0,
+        "window_pattern": "L",
+        "device_batch_size": 16,
+        "total_batch_size": 524288,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.008,
+        "weight_decay": 0.28,
+        "matrix_lr": 0.02,
+        "scalar_lr": 0.5,
+        "warmup_steps": 40,
+        "warmdown_ratio": 0.65,
+        "final_lr_frac": 0.05,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": 500,
+        "core_metric_max_per_task": 50,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-1ep-65sh-r30",
+        "group": "think-d12-stopping-point",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio30",
+          "65-shards",
+          "a100",
+          "bf16",
+          "stopping-point"
+        ]
+      },
+      "config_fingerprint": "35996219a51996ca",
+      "artifact_path": "experiments/think-d12-1ep-65sh-r30"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-1ep-65sh-r30",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "35996219a51996ca"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 23,
+    "pos": 59400769,
+    "epoch": 1,
+    "pq_idx": 23,
+    "rg_idx": 59400769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1270159041981165,
+    "smooth_train_loss": 2.9730753725244097,
+    "total_training_time": 12057.107451677322,
+    "stage_training_flops": 2092926498766848000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 2092926498766848000
+  }
+}