Import experiment archive from bart (batch 6)

experiments/think-d12-r11.25-ctx4096/tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53db25a1f6969198e67c303e479e97fb49859c715061baec525471666a2f47e1
+size 132649

experiments/think-d12-r11.25-ctx4096/tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42ea52179dcefb47e4a4bbca7c37f67c3373840a3681587b1c8fe188f00d3bf1
+size 404071

experiments/think-d12-r11.25-ctx8192/base_checkpoints/meta_000500.json

@@ -0,0 +1,140 @@
+{
+  "step": 500,
+  "training_complete": false,
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "val_bpb": 1.2806006574422366,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-r11.25-ctx8192",
+    "wandb_run_id": "e3483a4b",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio11.25,ctx8192",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 8192,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 4,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/base_checkpoints",
+    "experiment_id": "think-d12-r11.25-ctx8192",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/config.json",
+    "tokenizer_fingerprint": "03c4f62e7a9d0c3b",
+    "git_commit_sha": "9db3f1c2f998309198af5b020c869fd817d51847",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-r11.25-ctx8192",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-r11.25-ctx8192",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 24,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "max_seq_len": 8192,
+        "window_pattern": "L",
+        "device_batch_size": 4,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-r11.25-ctx8192",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio11.25",
+          "ctx8192"
+        ]
+      },
+      "config_fingerprint": "407a5074e0bf3730",
+      "artifact_path": "experiments/think-d12-r11.25-ctx8192"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-r11.25-ctx8192",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "407a5074e0bf3730"
+  },
+  "device_batch_size": 4,
+  "max_seq_len": 8192,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.2806006574422366,
+    "smooth_train_loss": 3.5198863114259193,
+    "total_training_time": 1859.6971344947815,
+    "stage_training_flops": 410668272451584000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 410668272451584000
+  }
+}

experiments/think-d12-r11.25-ctx8192/base_checkpoints/meta_001000.json

@@ -0,0 +1,140 @@
+{
+  "step": 1000,
+  "training_complete": false,
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "val_bpb": 1.1673074973592756,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-r11.25-ctx8192",
+    "wandb_run_id": "e3483a4b",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio11.25,ctx8192",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 8192,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 4,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/base_checkpoints",
+    "experiment_id": "think-d12-r11.25-ctx8192",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/config.json",
+    "tokenizer_fingerprint": "03c4f62e7a9d0c3b",
+    "git_commit_sha": "9db3f1c2f998309198af5b020c869fd817d51847",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-r11.25-ctx8192",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-r11.25-ctx8192",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 24,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "max_seq_len": 8192,
+        "window_pattern": "L",
+        "device_batch_size": 4,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-r11.25-ctx8192",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio11.25",
+          "ctx8192"
+        ]
+      },
+      "config_fingerprint": "407a5074e0bf3730",
+      "artifact_path": "experiments/think-d12-r11.25-ctx8192"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-r11.25-ctx8192",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "407a5074e0bf3730"
+  },
+  "device_batch_size": 4,
+  "max_seq_len": 8192,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.1673074973592756,
+    "smooth_train_loss": 3.291130702382907,
+    "total_training_time": 3762.280524253845,
+    "stage_training_flops": 821336544903168000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 821336544903168000
+  }
+}

experiments/think-d12-r11.25-ctx8192/base_checkpoints/meta_001500.json

@@ -0,0 +1,140 @@
+{
+  "step": 1500,
+  "training_complete": false,
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "val_bpb": 1.108596107059193,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-r11.25-ctx8192",
+    "wandb_run_id": "e3483a4b",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio11.25,ctx8192",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 8192,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 4,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/base_checkpoints",
+    "experiment_id": "think-d12-r11.25-ctx8192",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/config.json",
+    "tokenizer_fingerprint": "03c4f62e7a9d0c3b",
+    "git_commit_sha": "9db3f1c2f998309198af5b020c869fd817d51847",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-r11.25-ctx8192",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-r11.25-ctx8192",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 24,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "max_seq_len": 8192,
+        "window_pattern": "L",
+        "device_batch_size": 4,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-r11.25-ctx8192",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio11.25",
+          "ctx8192"
+        ]
+      },
+      "config_fingerprint": "407a5074e0bf3730",
+      "artifact_path": "experiments/think-d12-r11.25-ctx8192"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-r11.25-ctx8192",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "407a5074e0bf3730"
+  },
+  "device_batch_size": 4,
+  "max_seq_len": 8192,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86488769,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86488769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.108596107059193,
+    "smooth_train_loss": 3.1279163553930784,
+    "total_training_time": 5663.0339615345,
+    "stage_training_flops": 1232004817354752000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1232004817354752000
+  }
+}

experiments/think-d12-r11.25-ctx8192/base_checkpoints/meta_002000.json

@@ -0,0 +1,140 @@
+{
+  "step": 2000,
+  "training_complete": false,
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "val_bpb": 1.0589838913914653,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-r11.25-ctx8192",
+    "wandb_run_id": "e3483a4b",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio11.25,ctx8192",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 8192,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 4,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/base_checkpoints",
+    "experiment_id": "think-d12-r11.25-ctx8192",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/config.json",
+    "tokenizer_fingerprint": "03c4f62e7a9d0c3b",
+    "git_commit_sha": "9db3f1c2f998309198af5b020c869fd817d51847",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-r11.25-ctx8192",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-r11.25-ctx8192",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 24,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "max_seq_len": 8192,
+        "window_pattern": "L",
+        "device_batch_size": 4,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-r11.25-ctx8192",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio11.25",
+          "ctx8192"
+        ]
+      },
+      "config_fingerprint": "407a5074e0bf3730",
+      "artifact_path": "experiments/think-d12-r11.25-ctx8192"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-r11.25-ctx8192",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "407a5074e0bf3730"
+  },
+  "device_batch_size": 4,
+  "max_seq_len": 8192,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48640769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48640769
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0589838913914653,
+    "smooth_train_loss": 3.116437961262795,
+    "total_training_time": 7566.1544008255005,
+    "stage_training_flops": 1642673089806336000,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1642673089806336000
+  }
+}

experiments/think-d12-r11.25-ctx8192/base_checkpoints/meta_002362.json

@@ -0,0 +1,140 @@
+{
+  "step": 2362,
+  "training_complete": true,
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "val_bpb": 1.0395519592251896,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "think-d12-r11.25-ctx8192",
+    "wandb_run_id": "e3483a4b",
+    "wandb_group": "think-d12",
+    "wandb_tags": "think-dataset,d12,ratio11.25,ctx8192",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 8192,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 4,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": 2000,
+    "pretokenized": true,
+    "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/data",
+    "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/tokenizer",
+    "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/pretok",
+    "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/base_checkpoints",
+    "experiment_id": "think-d12-r11.25-ctx8192",
+    "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx8192/config.json",
+    "tokenizer_fingerprint": "03c4f62e7a9d0c3b",
+    "git_commit_sha": "9db3f1c2f998309198af5b020c869fd817d51847",
+    "seed": 42,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": "think-d12-r11.25-ctx8192",
+    "experiment": {
+      "schema_version": 1,
+      "stage": "base",
+      "experiment_id": "think-d12-r11.25-ctx8192",
+      "dataset": {
+        "adapter": "parquet_shards",
+        "repo": "jbduran/think-dataset",
+        "revision": "main",
+        "validation_shard": 472,
+        "num_train_shards": 24,
+        "download_workers": 4
+      },
+      "tokenizer": {
+        "mode": "train",
+        "max_chars": 2000000000,
+        "doc_cap": 10000,
+        "vocab_size": 32768
+      },
+      "pretokenize": {
+        "enabled": true,
+        "slack": 1.03,
+        "val_tokens": 20971520,
+        "shard_tokens": 100000000,
+        "tokenizer_threads": 8
+      },
+      "training": {
+        "depth": 12,
+        "scaling_params": 110100912,
+        "target_param_data_ratio": 11.25,
+        "max_seq_len": 8192,
+        "window_pattern": "L",
+        "device_batch_size": 4,
+        "total_batch_size": 524288,
+        "save_every": 500,
+        "eval_every": 250,
+        "eval_tokens": 2097152,
+        "core_metric_every": -1,
+        "sample_every": -1
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "name": "think-d12-r11.25-ctx8192",
+        "group": "think-d12",
+        "tags": [
+          "think-dataset",
+          "d12",
+          "ratio11.25",
+          "ctx8192"
+        ]
+      },
+      "config_fingerprint": "407a5074e0bf3730",
+      "artifact_path": "experiments/think-d12-r11.25-ctx8192"
+    },
+    "stage": "base",
+    "base_experiment_id": "think-d12-r11.25-ctx8192",
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "407a5074e0bf3730"
+  },
+  "device_batch_size": 4,
+  "max_seq_len": 8192,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 12,
+    "pos": 38471586,
+    "epoch": 1,
+    "pq_idx": 12,
+    "rg_idx": 38471586
+  },
+  "loop_state": {
+    "min_val_bpb": 1.0395519592251896,
+    "smooth_train_loss": 3.0155535492313272,
+    "total_training_time": 9008.03685593605,
+    "stage_training_flops": 1939996919061282816,
+    "inherited_parent_flops": 0.0,
+    "cumulative_pipeline_training_flops": 1939996919061282816
+  }
+}

experiments/think-d12-r11.25-ctx8192/base_checkpoints/model_000500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51922864ab7ccea4cd40458bb60898167d02696982c1e4c0de684995e5f26290
+size 792761690

experiments/think-d12-r11.25-ctx8192/base_checkpoints/model_001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8c23459c7bef825a21a0236f2b9e2efba66c8b427c78a7a6cca852df957fd0e
+size 792761690

experiments/think-d12-r11.25-ctx8192/base_checkpoints/model_001500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dd6cc545ba54787b342e190d75e857872d1964f10a4032ab0d49640012f84b7
+size 792761690

experiments/think-d12-r11.25-ctx8192/base_checkpoints/model_002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e6c50126914bc4d20f9db6222891ee9ee61c634634b023a13e5f1e583e9403
+size 792761690

experiments/think-d12-r11.25-ctx8192/base_checkpoints/model_002362.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43a07973a6d23613f28d1b43623e06bba393623fd88452e1c495bf26aa9ac4d6
+size 792761690

experiments/think-d12-r11.25-ctx8192/base_checkpoints/optim_000500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f402cf80bf9d4b917ae2a99240b0ea34cd842f025927c148bccf384ce9b744b8
+size 1246165357

experiments/think-d12-r11.25-ctx8192/base_checkpoints/optim_001000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ef8f97be5f4890871d5586afd42bc946b9058ae996deaa83c14c8f71610deb9
+size 1246165357

experiments/think-d12-r11.25-ctx8192/base_checkpoints/optim_001500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df9bf4a59d46c126bd65f5cdc6de8fb531c492fabf70f9db22a5ac27bd08dd2f
+size 1246165357

experiments/think-d12-r11.25-ctx8192/base_checkpoints/optim_002000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f95fc4fe7c0a841f3e443102d00495e4f4305ea955ad2b81ffd3ad8865cbd34e
+size 1246165357

experiments/think-d12-r11.25-ctx8192/base_checkpoints/optim_002362_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e1464e581c921562375dd5957b76dad9fe37d04a17e9a021a7876026e624044
+size 1246165357

experiments/think-d12-r11.25-ctx8192/config.json

@@ -0,0 +1,57 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 24,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 12,
+    "scaling_params": 110100912,
+    "target_param_data_ratio": 11.25,
+    "max_seq_len": 8192,
+    "window_pattern": "L",
+    "device_batch_size": 4,
+    "total_batch_size": 524288,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano"
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "think-d12-r11.25-ctx8192",
+    "group": "think-d12",
+    "tags": [
+      "think-dataset",
+      "d12",
+      "ratio11.25",
+      "ctx8192"
+    ]
+  },
+  "config_fingerprint": "407a5074e0bf3730",
+  "artifact_path": "experiments/think-d12-r11.25-ctx8192"
+}

experiments/think-d12-r11.25-ctx8192/evals/core.json

@@ -0,0 +1,56 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": 0.07251341817663091,
+  "core_results": {
+    "hellaswag_zeroshot": 0.2757418751716614,
+    "jeopardy": 0.0009447330958209932,
+    "bigbench_qa_wikidata": 0.0832144096493721,
+    "arc_easy": 0.3194444477558136,
+    "arc_challenge": 0.21501706540584564,
+    "copa": 0.5099999904632568,
+    "commonsense_qa": 0.31285831332206726,
+    "piqa": 0.5331882238388062,
+    "openbook_qa": 0.24800001084804535,
+    "lambada_openai": 0.23423248529434204,
+    "hellaswag": 0.2802230417728424,
+    "winograd": 0.5714285969734192,
+    "winogrande": 0.4956590235233307,
+    "bigbench_dyck_languages": 0.10200000554323196,
+    "agi_eval_lsat_ar": 0.260869562625885,
+    "bigbench_cs_algorithms": 0.41969695687294006,
+    "bigbench_operators": 0.07619047909975052,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.024030273780226707,
+    "coqa": 0.0821746215224266,
+    "boolq": 0.5590214133262634,
+    "bigbench_language_identification": 0.2524999976158142
+  },
+  "centered_results": {
+    "hellaswag_zeroshot": 0.034322500228881836,
+    "jeopardy": 0.0009447330958209932,
+    "bigbench_qa_wikidata": 0.0832144096493721,
+    "arc_easy": 0.09259259700775146,
+    "arc_challenge": -0.04664391279220581,
+    "copa": 0.019999980926513672,
+    "commonsense_qa": 0.14107289165258405,
+    "piqa": 0.0663764476776123,
+    "openbook_qa": -0.002666652202606201,
+    "lambada_openai": 0.23423248529434204,
+    "hellaswag": 0.04029738903045654,
+    "winograd": 0.14285719394683838,
+    "winogrande": -0.008681952953338623,
+    "bigbench_dyck_languages": 0.10200000554323196,
+    "agi_eval_lsat_ar": 0.07608695328235625,
+    "bigbench_cs_algorithms": 0.41969695687294006,
+    "bigbench_operators": 0.07619047909975052,
+    "bigbench_repeat_copy_logic": 0.0,
+    "squad": 0.024030273780226707,
+    "coqa": 0.0821746215224266,
+    "boolq": -0.1604699649308857,
+    "bigbench_language_identification": 0.177667764153811
+  },
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-r11.25-ctx8192/evals/samples.json

@@ -0,0 +1,48 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is not yet fully developed. The capital of the United States is not yet fully developed"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the symbol of the gold, and the symbol of the silver. The gold is"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be the day of the week. \n\nThe day of the week is the same as"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the same as hot. \n\nThe hot is the same as hot. \n\nThe"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, 2. The moon, 3. The"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the color of the skin of the face, and the color of the skin."
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the number of the number of the number of the number of the"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>ALIENS. A concern called industrial, in which all miners of ability for useful labor were engaged in obtaining industrial materials for making snuffers. Although it would be a rude and untenable enterprise to make different classes of miners dispose of goods for profit, different miners differing between the quality of the material misspelled and its quantity, it always staggers the mind with the idea of the matter which it concerns.\u00b9 Four or five shopkeepers are seen at so many tradeshops in town near together in towns and villages. \n\nMoney is better paid to supply the needs of skilled men than it is in overcrowded",
+    "<|bos|>37374.31 617.471.11 379.75 \n\nSwinburne, Jes. 10, 335.\n\nStatistical Index. \n\nCoates, erance, 1333. \n\nPurple Debenture, 1460.\n\nGenetic Index. \n\nSwinfenning, 51858.29, 1319. \n\nFree Presses. \n\nRidgens-Pantrepous, 25. \n\nSprayl-Power, regular exercise, 1700. \n\nPreachers and Teachers of the Schools, 401\u20132. \n\n",
+    "<|bos|>URE FOOD AND THE BODY' \n\nIn such a commonwealth Siamese readers might find in Kumber's Essays, or Mabon's Vol. of Tobit and St. Jerome's Lives, a sound, a clear and satisfactory explanation of this phrase. May Lady Cassius inform the reverend Society from which this paragraph is borrowed that in this hour of peril men and women may \"paint to him,\" and bewail Rest and Treatment. In both of these circles there are three main meanings attached to the phrase. One is, with the excessive reference in hotel-keepers, donkeys, or hares; another, with alligators",
+    "<|bos|>HENRY MART 140. \n\nLeblay, Mr. De Martyn's invention of music, 4. IX.\n\nTRANSLATOR'S NOTE.-Send forth a translation of the Notes which were received by me, translated from the Musical \n\nCommission's Calendar.\n\nHis performance we cannot altogether estimate, but St. Columba gave us confirmation of his inventions, 30. XXVIIii, 18. Eh, What (Georgics, I, 177); 'Slightest book that ever was written' (Sonn., lies 28\u00bd), a work of high merit read with honour,\n\nQu\u00e6rese",
+    "<|bos|>Harvard School, IV Department of Education, 1843-1972. \n\n2 Henry State League, LL. concerning Courses in Medicine and the Arts, pp. 22 et seq.\n\nHistory of the Monroe Doctrine, by one who has visited Europe, compiled from European Authorities.\n\nNew York: N. Y. \n\nExaminer, Vol. XXXI, \n\nApril, 1917, p. 88.\n\nPamphlet on Scien tific Methods of Education (\"Outlines of the Maladies and Defects of the Methods of Industrial Society,\" by Dr. Jevons). \n\nNew York, September and October, 19",
+    "<|bos|>The Bird reflects upon. his. \n\nThe Clipper. \n\nVapour.\n\nUntil within a few weeks the admission of the truth to our beloved Bird was fatal to that race, little cared for neither in her recorded history nor since they married, and still less as regards her character. Her reign ended; and when she died only after a few months good for nothing the country felt herself well restored to health. She began now to see her way. Our dear bird became as dear to her as the Christian mother; she began to see her way clearer to her senses; and as her thoughts turned, and freedom fell back, she",
+    "<|bos|>Army of the Cumberland and Arkansas Army, and an Army of the Potomac under the command of Martin Robertson.\n\nHEADQUARTERS CAMP THIRDQUARTERS, THIRD BRIG 1ST BRIG 1ST BRIG 1ST BRIG 1ST BRIG \n\n6 8 8 \n\nMCLQUERISHER'S STATION, 9 P.M. \n\nMY ARMY, CAL. \n\nEnlarged with orders by the War Department.\n\nHeadquarters Camp War Department, Clope Ridge, Va., September 15, 1864. \n\n6 P.M The Confederates tend S. M. Camp are in the Confederate service hospital at N. C.",
+    "<|bos|>the eightieth year of his age.\n\nI had never been conferring, like the palette and the gallens at which I used to sit. never had thought it wrong to give the faintest hint of this prudery, which I trust is always requested of the upholder at his housekeeping, as shall appear by the order and directions accompanying it. So while I was listening to the wise old voice of the tender bride calling alone in her measure that nonsense of patriarchal impiety. It was all the more gratifying when I heard that the Governor of Shetton is now apparently labouring in the same breath, when he speaks"
+  ]
+}

experiments/think-d12-r11.25-ctx8192/evals/val_bpb.json

@@ -0,0 +1,174 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val_per_position": [
+      {
+        "start": 0,
+        "end": 256,
+        "bpb": 1.1345637945630735
+      },
+      {
+        "start": 256,
+        "end": 512,
+        "bpb": 1.0678589586934193
+      },
+      {
+        "start": 512,
+        "end": 768,
+        "bpb": 1.0513444442729074
+      },
+      {
+        "start": 768,
+        "end": 1024,
+        "bpb": 1.0420207610099703
+      },
+      {
+        "start": 1024,
+        "end": 1280,
+        "bpb": 1.0363256199642377
+      },
+      {
+        "start": 1280,
+        "end": 1536,
+        "bpb": 1.0293503892901525
+      },
+      {
+        "start": 1536,
+        "end": 1792,
+        "bpb": 1.0270513457476949
+      },
+      {
+        "start": 1792,
+        "end": 2048,
+        "bpb": 1.0255301775348173
+      },
+      {
+        "start": 2048,
+        "end": 2304,
+        "bpb": 1.0171632048394768
+      },
+      {
+        "start": 2304,
+        "end": 2560,
+        "bpb": 1.0150940036004366
+      },
+      {
+        "start": 2560,
+        "end": 2816,
+        "bpb": 1.0132597834490384
+      },
+      {
+        "start": 2816,
+        "end": 3072,
+        "bpb": 1.0154303287433495
+      },
+      {
+        "start": 3072,
+        "end": 3328,
+        "bpb": 1.0125085420227948
+      },
+      {
+        "start": 3328,
+        "end": 3584,
+        "bpb": 1.0117220799135607
+      },
+      {
+        "start": 3584,
+        "end": 3840,
+        "bpb": 1.0093052242865306
+      },
+      {
+        "start": 3840,
+        "end": 4096,
+        "bpb": 1.0067290549863526
+      },
+      {
+        "start": 4096,
+        "end": 4352,
+        "bpb": 1.007627760298138
+      },
+      {
+        "start": 4352,
+        "end": 4608,
+        "bpb": 1.005475773581573
+      },
+      {
+        "start": 4608,
+        "end": 4864,
+        "bpb": 1.0011348016292028
+      },
+      {
+        "start": 4864,
+        "end": 5120,
+        "bpb": 1.0025369118565095
+      },
+      {
+        "start": 5120,
+        "end": 5376,
+        "bpb": 0.9978363303279962
+      },
+      {
+        "start": 5376,
+        "end": 5632,
+        "bpb": 0.9936016054109623
+      },
+      {
+        "start": 5632,
+        "end": 5888,
+        "bpb": 0.9930789448128515
+      },
+      {
+        "start": 5888,
+        "end": 6144,
+        "bpb": 0.988120986726172
+      },
+      {
+        "start": 6144,
+        "end": 6400,
+        "bpb": 0.9878455863729801
+      },
+      {
+        "start": 6400,
+        "end": 6656,
+        "bpb": 0.988322568304946
+      },
+      {
+        "start": 6656,
+        "end": 6912,
+        "bpb": 0.9895607000315525
+      },
+      {
+        "start": 6912,
+        "end": 7168,
+        "bpb": 0.9924135354945043
+      },
+      {
+        "start": 7168,
+        "end": 7424,
+        "bpb": 0.9887797597227126
+      },
+      {
+        "start": 7424,
+        "end": 7680,
+        "bpb": 0.9863644464805957
+      },
+      {
+        "start": 7680,
+        "end": 7936,
+        "bpb": 0.9852729515784135
+      },
+      {
+        "start": 7936,
+        "end": 8192,
+        "bpb": 0.9838855089135268
+      }
+    ],
+    "val": 1.0127322889400117
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-r11.25-ctx8192/run.json

@@ -0,0 +1,10 @@
+{
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "stage": "base",
+  "base_experiment_id": "think-d12-r11.25-ctx8192",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "config_fingerprint": "407a5074e0bf3730",
+  "wandb_run_id": "e3483a4b",
+  "created_at": 1783693257
+}

experiments/think-d12-r11.25-ctx8192/summary.json

@@ -0,0 +1,69 @@
+{
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "stage": "base",
+  "base_experiment_id": "think-d12-r11.25-ctx8192",
+  "parent_experiment_id": null,
+  "parent_checkpoint_step": null,
+  "dataset": "jbduran/think-dataset",
+  "dataset_revision": "main",
+  "step": 2362,
+  "depth": 12,
+  "target_param_data_ratio": 11.25,
+  "training_tokens": 1238368256,
+  "final_sampled_val_bpb": 1.0395519592251896,
+  "minimum_sampled_val_bpb": 1.0395519592251896,
+  "full_val_bpb": 1.0127322889400117,
+  "core_metric": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is not yet fully developed. The capital of the United States is not yet fully developed"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the symbol of the gold, and the symbol of the silver. The gold is"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be the day of the week. \n\nThe day of the week is the same as"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the same as hot. \n\nThe hot is the same as hot. \n\nThe"
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, 2. The moon, 3. The"
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the color of the skin of the face, and the color of the skin."
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the number of the number of the number of the number of the"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>ALIENS. A concern called industrial, in which all miners of ability for useful labor were engaged in obtaining industrial materials for making snuffers. Although it would be a rude and untenable enterprise to make different classes of miners dispose of goods for profit, different miners differing between the quality of the material misspelled and its quantity, it always staggers the mind with the idea of the matter which it concerns.\u00b9 Four or five shopkeepers are seen at so many tradeshops in town near together in towns and villages. \n\nMoney is better paid to supply the needs of skilled men than it is in overcrowded",
+    "<|bos|>37374.31 617.471.11 379.75 \n\nSwinburne, Jes. 10, 335.\n\nStatistical Index. \n\nCoates, erance, 1333. \n\nPurple Debenture, 1460.\n\nGenetic Index. \n\nSwinfenning, 51858.29, 1319. \n\nFree Presses. \n\nRidgens-Pantrepous, 25. \n\nSprayl-Power, regular exercise, 1700. \n\nPreachers and Teachers of the Schools, 401\u20132. \n\n",
+    "<|bos|>URE FOOD AND THE BODY' \n\nIn such a commonwealth Siamese readers might find in Kumber's Essays, or Mabon's Vol. of Tobit and St. Jerome's Lives, a sound, a clear and satisfactory explanation of this phrase. May Lady Cassius inform the reverend Society from which this paragraph is borrowed that in this hour of peril men and women may \"paint to him,\" and bewail Rest and Treatment. In both of these circles there are three main meanings attached to the phrase. One is, with the excessive reference in hotel-keepers, donkeys, or hares; another, with alligators",
+    "<|bos|>HENRY MART 140. \n\nLeblay, Mr. De Martyn's invention of music, 4. IX.\n\nTRANSLATOR'S NOTE.-Send forth a translation of the Notes which were received by me, translated from the Musical \n\nCommission's Calendar.\n\nHis performance we cannot altogether estimate, but St. Columba gave us confirmation of his inventions, 30. XXVIIii, 18. Eh, What (Georgics, I, 177); 'Slightest book that ever was written' (Sonn., lies 28\u00bd), a work of high merit read with honour,\n\nQu\u00e6rese",
+    "<|bos|>Harvard School, IV Department of Education, 1843-1972. \n\n2 Henry State League, LL. concerning Courses in Medicine and the Arts, pp. 22 et seq.\n\nHistory of the Monroe Doctrine, by one who has visited Europe, compiled from European Authorities.\n\nNew York: N. Y. \n\nExaminer, Vol. XXXI, \n\nApril, 1917, p. 88.\n\nPamphlet on Scien tific Methods of Education (\"Outlines of the Maladies and Defects of the Methods of Industrial Society,\" by Dr. Jevons). \n\nNew York, September and October, 19",
+    "<|bos|>The Bird reflects upon. his. \n\nThe Clipper. \n\nVapour.\n\nUntil within a few weeks the admission of the truth to our beloved Bird was fatal to that race, little cared for neither in her recorded history nor since they married, and still less as regards her character. Her reign ended; and when she died only after a few months good for nothing the country felt herself well restored to health. She began now to see her way. Our dear bird became as dear to her as the Christian mother; she began to see her way clearer to her senses; and as her thoughts turned, and freedom fell back, she",
+    "<|bos|>Army of the Cumberland and Arkansas Army, and an Army of the Potomac under the command of Martin Robertson.\n\nHEADQUARTERS CAMP THIRDQUARTERS, THIRD BRIG 1ST BRIG 1ST BRIG 1ST BRIG 1ST BRIG \n\n6 8 8 \n\nMCLQUERISHER'S STATION, 9 P.M. \n\nMY ARMY, CAL. \n\nEnlarged with orders by the War Department.\n\nHeadquarters Camp War Department, Clope Ridge, Va., September 15, 1864. \n\n6 P.M The Confederates tend S. M. Camp are in the Confederate service hospital at N. C.",
+    "<|bos|>the eightieth year of his age.\n\nI had never been conferring, like the palette and the gallens at which I used to sit. never had thought it wrong to give the faintest hint of this prudery, which I trust is always requested of the upholder at his housekeeping, as shall appear by the order and directions accompanying it. So while I was listening to the wise old voice of the tender bride calling alone in her measure that nonsense of patriarchal impiety. It was all the more gratifying when I heard that the Governor of Shetton is now apparently labouring in the same breath, when he speaks"
+  ],
+  "training_time_seconds": 9008.03685593605,
+  "stage_training_flops": 1.9399969190612828e+18,
+  "inherited_parent_flops": 0.0,
+  "cumulative_pipeline_training_flops": 1.9399969190612828e+18,
+  "config_fingerprint": "407a5074e0bf3730",
+  "git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
+  "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/e3483a4b",
+  "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-r11.25-ctx8192",
+  "dataset_fingerprint": "63a5e6be81591d82",
+  "tokenizer_fingerprint": "03c4f62e7a9d0c3b",
+  "unique_train_tokens": 0
+}

experiments/think-d12-r11.25-ctx8192/tokenizer/experiment_tokenizer.json

@@ -0,0 +1,18 @@
+{
+  "experiment_id": "think-d12-r11.25-ctx8192",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 24,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "created_at": 1783708207
+}

experiments/think-d12-r11.25-ctx8192/tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53db25a1f6969198e67c303e479e97fb49859c715061baec525471666a2f47e1
+size 132649

experiments/think-d12-r11.25-ctx8192/tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42ea52179dcefb47e4a4bbca7c37f67c3373840a3681587b1c8fe188f00d3bf1
+size 404071

experiments/think-d12-r11.25/base_checkpoints/meta_000500.json

@@ -0,0 +1,61 @@
+{
+  "step": 500,
+  "val_bpb": null,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": -1,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "eval_every": -1,
+    "eval_tokens": 41943040,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": null
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 2,
+    "pos": 62184769,
+    "epoch": 1,
+    "pq_idx": 2,
+    "rg_idx": 62184769
+  },
+  "loop_state": {
+    "min_val_bpb": Infinity,
+    "smooth_train_loss": 3.5893779623775406,
+    "total_training_time": 1290.4532148838043
+  }
+}

experiments/think-d12-r11.25/base_checkpoints/meta_001000.json

@@ -0,0 +1,61 @@
+{
+  "step": 1000,
+  "val_bpb": null,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": -1,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "eval_every": -1,
+    "eval_tokens": 41943040,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": null
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 5,
+    "pos": 24336769,
+    "epoch": 1,
+    "pq_idx": 5,
+    "rg_idx": 24336769
+  },
+  "loop_state": {
+    "min_val_bpb": Infinity,
+    "smooth_train_loss": 3.4419165825253133,
+    "total_training_time": 2609.577807664871
+  }
+}

experiments/think-d12-r11.25/base_checkpoints/meta_001500.json

@@ -0,0 +1,61 @@
+{
+  "step": 1500,
+  "val_bpb": null,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": -1,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "eval_every": -1,
+    "eval_tokens": 41943040,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": null
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 7,
+    "pos": 86488769,
+    "epoch": 1,
+    "pq_idx": 7,
+    "rg_idx": 86488769
+  },
+  "loop_state": {
+    "min_val_bpb": Infinity,
+    "smooth_train_loss": 3.2465426140603015,
+    "total_training_time": 3929.598204135895
+  }
+}

experiments/think-d12-r11.25/base_checkpoints/meta_002000.json

@@ -0,0 +1,61 @@
+{
+  "step": 2000,
+  "val_bpb": null,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": -1,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "eval_every": -1,
+    "eval_tokens": 41943040,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": null
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 10,
+    "pos": 48640769,
+    "epoch": 1,
+    "pq_idx": 10,
+    "rg_idx": 48640769
+  },
+  "loop_state": {
+    "min_val_bpb": Infinity,
+    "smooth_train_loss": 3.2442641345309373,
+    "total_training_time": 5249.494728565216
+  }
+}

experiments/think-d12-r11.25/base_checkpoints/meta_002362.json

@@ -0,0 +1,61 @@
+{
+  "step": 2362,
+  "val_bpb": null,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 12,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "L",
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 11.25,
+    "device_batch_size": 16,
+    "total_batch_size": -1,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "pretokenized": true,
+    "eval_every": -1,
+    "eval_tokens": 41943040,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 500,
+    "model_tag": null
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "file_idx": 12,
+    "pos": 38438817,
+    "epoch": 1,
+    "pq_idx": 12,
+    "rg_idx": 38438817
+  },
+  "loop_state": {
+    "min_val_bpb": Infinity,
+    "smooth_train_loss": 3.074421420856799,
+    "total_training_time": 6205.646646976471
+  }
+}

experiments/think-d12-r11.25/base_checkpoints/model_000500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21fbdff366e5db3fa2f254b4b98c763cc70ba95722242fa032d8d21b956b694f
+size 792761399

experiments/think-d12-r11.25/base_checkpoints/model_001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f0a862044e51b8cac250cf1e4f26af7f8347fd887b8c30af08addb879b6494d
+size 792761399

experiments/think-d12-r11.25/base_checkpoints/model_001500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:326bddc26c100de33310f9164dc873af6099a9b7760527a8707c256988a8ec7a
+size 792761399

experiments/think-d12-r11.25/base_checkpoints/model_002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5d101975722cffc43eb4f35aa967a5afd397b159da3521e5a1acbd3819d466e
+size 792761399

experiments/think-d12-r11.25/base_checkpoints/model_002362.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04331c52ed8fa7259f350e4ec72d0dd6602451cfd75a5773a4c17ac5c141ea7e
+size 792761399

experiments/think-d12-r11.25/base_checkpoints/optim_000500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39a131089a476a202ae932f21d9398b225d0b8e4147a58fbdff797914d34976
+size 1246165237

experiments/think-d12-r11.25/base_checkpoints/optim_001000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4971566eb3d4aa5d5fe29b3a3b77f56581b61713e5c1d162debb8a409c02118
+size 1246165237

experiments/think-d12-r11.25/base_checkpoints/optim_001500_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05ce87d44bfd6e9b4d4c1e643a2a5aa4b169119913ccbdf3625d7cfa7313b342
+size 1246165237

experiments/think-d12-r11.25/base_checkpoints/optim_002000_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b58f68fd887360ce99e8456aecf706b1bcf5c8210194721389698ec658237b1
+size 1246165237

experiments/think-d12-r11.25/base_checkpoints/optim_002362_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09aa5632a4b33981a1b2fbd97254d0050ecb7916d0c242f1d195c0726be314d7
+size 1246165237

experiments/think-d12-r11.25/config.json

@@ -0,0 +1,55 @@
+{
+  "schema_version": 1,
+  "stage": "base",
+  "experiment_id": "think-d12-r11.25",
+  "dataset": {
+    "adapter": "parquet_shards",
+    "repo": "jbduran/think-dataset",
+    "revision": "main",
+    "validation_shard": 472,
+    "num_train_shards": 24,
+    "download_workers": 4
+  },
+  "tokenizer": {
+    "mode": "train",
+    "max_chars": 2000000000,
+    "doc_cap": 10000,
+    "vocab_size": 32768
+  },
+  "pretokenize": {
+    "enabled": true,
+    "slack": 1.03,
+    "val_tokens": 20971520,
+    "shard_tokens": 100000000,
+    "tokenizer_threads": 8
+  },
+  "training": {
+    "depth": 12,
+    "scaling_params": 110100912,
+    "target_param_data_ratio": 11.25,
+    "window_pattern": "L",
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "save_every": 500,
+    "eval_every": 250,
+    "eval_tokens": 2097152,
+    "core_metric_every": -1,
+    "sample_every": -1
+  },
+  "artifacts": {
+    "repo": "jbduran/think.nano"
+  },
+  "wandb": {
+    "entity": "jbduran-thinkingmachinesncsu",
+    "project": "think.nano",
+    "name": "think-d12-r11.25",
+    "group": "think-d12",
+    "tags": [
+      "think-dataset",
+      "d12",
+      "ratio11.25"
+    ]
+  },
+  "config_fingerprint": "3b5a68714770b6af",
+  "artifact_path": "experiments/think-d12-r11.25"
+}

experiments/think-d12-r11.25/evals/samples.json

@@ -0,0 +1,48 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {},
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [
+    {
+      "prompt": "The capital of France is",
+      "text": "<|bos|>The capital of France is 10,000,000 francs, and the capital of the United"
+    },
+    {
+      "prompt": "The chemical symbol of gold is",
+      "text": "<|bos|>The chemical symbol of gold is the gold of the \n\nUnited States. It is the gold of the United States"
+    },
+    {
+      "prompt": "If yesterday was Friday, then tomorrow will be",
+      "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nThe day is Sunday, and the day is Sunday. \n\nThe"
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe opposite of cold is the opposite of cold."
+    },
+    {
+      "prompt": "The planets of the solar system are:",
+      "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the stars. \n\n2."
+    },
+    {
+      "prompt": "My favorite color is",
+      "text": "<|bos|>My favorite color is the color of the sky. \n\nThe color of the sky is a color of"
+    },
+    {
+      "prompt": "If 5*x + 3 = 13, then x is",
+      "text": "<|bos|>If 5*x + 3 = 13, then x is the number of times the number of times the number of times the number of times"
+    }
+  ],
+  "unconditioned_samples": [
+    "<|bos|>Monthillahivan-this is worthy, they say, of the glory of God's presence, our Father being now to come again! \n\nDead. The priest who said Lord! so between thy guilty hands exulting and curse shoot; do thy honors sweetly, O Father! touch me not with panic, nor miss me with holy surprise, nor sigh (say) tost away my days, nor desolate them; thou wert sent by heaven, and seen with so much care, that Thou near thy sins didst repair them; how art Thou now to come thus early, and bear unto me so divinely? not",
+    "<|bos|>370 \n\nPaxton's Introduction to Education, or the true philosophy of the schools criticised, is differentiated. In Rossetti's plan, as already noted, it formulated in these brief articles, there should be but two different versions of the 73\n\n-flat elements; in Rossetti's, we commonly use the broad current of their general aim. That which has been called a positive and material-is aptly called a negative-was given with corresponding emphasis at three different times, at different epochs, but the THREE are frequently gods regular in origin, and preserved from the violence of adjustment and shift, rather than dormant and deliberate meaning of",
+    "<|bos|>ROrepoys and Willieot's book on the Siamese Law. \n\nA Letter from Frank Perrivsky to a Prince of Wales. By Kinsman Keith. \n\nTHE \n\nMAY, 1923. \n\nIn Paper \n\nWith 25 Illustrations. Parts. $2 $9 $14 6 $2.50 \n\nIN POLITICAL IDOLSES. \n\nAmerican Law. By George W. Resting, LL.D., Professor of Political Economy in Princeton University. \n\n$18 $2.50 \n\nSamson Minot, hotel-keeper. $3 $7\n\nTHE SCALE OF SIZE.",
+    "<|bos|>HENRY MARTYN SAXON, THE HINDUS CHRISTURIENT. \n\nEDWARD IRVING, of the College of the Anatomy School of \n\nDurham, Surrey.\n\nEDWARD PERCY BAKER, OF ALICE COLLEGE, whose personal appearance is now in print, was born at Stanneley, Surrey, July 18, 1794. He has been student in the Company's Military College at Woolwich for more than one year, for his learning and industry in his profession and studies. He has written a book entitled History, Economics and Political Science, which lies nearly at our very door, entitled History, Political Science and Political \n\nScience. It is",
+    "<|bos|> HOUSE OF THE ANGELS. \n\nFrom the City of St. Ann. \n\n2 vols. 3s. My Last in a Garden. I reserve for the fifth edition, in manuscript, a full account of ancient His tory. In \n\n1 vol. 5, a short history of Nero and Herod. from 6 to \n\n10 Years, both kept in this Library.\n\n4 \n\nLibrary of the Inducci EN QUANDRON. \n\nFRONTIER, SAMUEL, Dean of Carlisle, President of the\n\nAmerican Board of Works, 3 vols. 2 vols. 3s. 6d. \n\nLibrary",
+    "<|bos|>The Bird reflects the world, and swims the eagle's web.-Van Isle.\n\nSHOP'S \n\nREACH \n\nNEGARYELY GENIAL SHOP FINANCIERS RAIDER \n\nREAR HEADS,.} \n\nREPUBLICS, WHILE SHE UNDER FULLER'S PORT, \n\nREP Philosophers, that they be not \n\nPharaoh's patterns good for nothing, and valiant men for that which is nothing; \n\nCLY VAUS\u00d2 EXOVENT \u03b4\u1f72 \u03bf\u03cd\u03c3\u03b1\u03b9 \u0391\u03b4\u03af\u03b4\u03b5\u03b9ANTA\u03c1, \u1f15\u03b4\u03b1\u03c1\u03c9\u03bd \u03c3\u03bf\u03c5\u03b4\u03bf\u1f7a\u03c2 The Queen eschews",
+    "<|bos|>'Eau du Monarch beau ou H\u00f4tel de Voodjiches.\" (The same French Inspecteur :) \"Son jours\n\nA \u00e9t\u00e9 autant \u00e0 tomboi les m\u00eames Premi\u00e8res avec une princesse qui sont d'ombres le miraculeur bien de France le souffrir. Les c\u00f4tes de ceux qui le sont j\u00e9sibres.\" (The French Directors.)\n\neffected these river improvements in effeminate cases, in the The Executive has eschewed the corrupdemell\u00e8 hrs. Nieuw Ga",
+    "<|bos|>the eight (250) series.\n\nThe pressure is already great.\n\nashions like the palette de la \n\nLast chapter, however, it may be noted, and it is a most common practice in such a period for small and countless series to transform the palette de la up half at the sauce, and it is a matter of considerable importance to seeing the clothes-holders uniformly the marks that indicate the supply of the palette de la into that respectable width whence their sulky tints are generally returned. \n\nThe colour represents the price of the garment at the time apparently labouring in the work. Delivery is possible"
+  ]
+}

experiments/think-d12-r11.25/evals/val_bpb.json

@@ -0,0 +1,54 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val_per_position": [
+      {
+        "start": 0,
+        "end": 256,
+        "bpb": 1.1255410237731462
+      },
+      {
+        "start": 256,
+        "end": 512,
+        "bpb": 1.0636677720059344
+      },
+      {
+        "start": 512,
+        "end": 768,
+        "bpb": 1.0506832286096848
+      },
+      {
+        "start": 768,
+        "end": 1024,
+        "bpb": 1.0456638664028364
+      },
+      {
+        "start": 1024,
+        "end": 1280,
+        "bpb": 1.038324560694976
+      },
+      {
+        "start": 1280,
+        "end": 1536,
+        "bpb": 1.0330914959872517
+      },
+      {
+        "start": 1536,
+        "end": 1792,
+        "bpb": 1.0308104068466784
+      },
+      {
+        "start": 1792,
+        "end": 2048,
+        "bpb": 1.027579795687179
+      }
+    ],
+    "val": 1.0519195678472355
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-r11.25/evals/val_bpb_on_think-dataset-clean-1930s.json

@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val": 1.0781689302026238
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-r11.25/evals/val_bpb_on_think-dataset-clean.json

@@ -0,0 +1,12 @@
+{
+  "model": "base_model (step 2362)",
+  "step": 2362,
+  "bpb": {
+    "val": 1.081385374786877
+  },
+  "core_metric": null,
+  "core_results": null,
+  "centered_results": null,
+  "conditioned_samples": [],
+  "unconditioned_samples": []
+}

experiments/think-d12-r11.25/run.json

@@ -0,0 +1,6 @@
+{
+  "experiment_id": "think-d12-r11.25",
+  "stage": "base",
+  "wandb_run_id": null,
+  "migration_note": "Migrated from the pre-lineage repository layout."
+}

experiments/think-d12-r11.25/sft/smoltalk-mmlu3-gsm8k4-v1/checkpoints/meta_001065.json

@@ -0,0 +1,38 @@
+{
+  "step": 1065,
+  "val_bpb": 0.39271438585109175,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 12,
+    "n_head": 6,
+    "n_kv_head": 6,
+    "n_embd": 768,
+    "window_pattern": "L"
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "model_tag": "d12",
+    "model_step": null,
+    "load_optimizer": 1,
+    "num_iterations": -1,
+    "max_seq_len": null,
+    "device_batch_size": 8,
+    "total_batch_size": null,
+    "embedding_lr": null,
+    "unembedding_lr": null,
+    "matrix_lr": null,
+    "init_lr_frac": 0.8,
+    "warmup_ratio": 0.0,
+    "warmdown_ratio": 0.5,
+    "final_lr_frac": 0.0,
+    "eval_every": -1,
+    "eval_tokens": 20971520,
+    "chatcore_every": -1,
+    "chatcore_max_cat": -1,
+    "chatcore_max_sample": 24,
+    "mmlu_epochs": 3,
+    "gsm8k_epochs": 4
+  }
+}

experiments/think-d12-r11.25/sft/smoltalk-mmlu3-gsm8k4-v1/checkpoints/model_001065.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ef8886ea2cbd820baa9bc98368f99673efb1f5fdbd082196d573b291111bda
+size 792761399

experiments/think-d12-r11.25/sft/smoltalk-mmlu3-gsm8k4-v1/checkpoints/optim_001065_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db8c9df6e288618ed6362102e29edc3731267b4ff382154a55709b4d5a14f1d4
+size 1246165237