File size: 7,229 Bytes
6951e66 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | {
"step": 2000,
"training_complete": false,
"experiment_id": "Think.Unbounded-d32",
"parent_experiment_id": null,
"parent_checkpoint_step": null,
"val_bpb": 0.9928518912323213,
"model_config": {
"sequence_len": 4096,
"vocab_size": 32768,
"n_layer": 32,
"n_head": 16,
"n_kv_head": 16,
"n_embd": 2048,
"window_pattern": "SSSL"
},
"user_config": {
"run": "Think.Unbounded-d32",
"wandb_run_id": "9b75691b",
"wandb_group": "clean1930s-d32",
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
"device_type": "",
"fp8": true,
"fp8_recipe": "tensorwise",
"depth": 32,
"aspect_ratio": 64,
"head_dim": 128,
"max_seq_len": 4096,
"window_pattern": "SSSL",
"num_iterations": 9600,
"target_flops": -1.0,
"target_param_data_ratio": -1.0,
"device_batch_size": 2,
"total_batch_size": 2097152,
"embedding_lr": 0.3,
"unembedding_lr": 0.008,
"weight_decay": 0.28,
"matrix_lr": 0.02,
"muon_momentum": 0.9,
"scalar_lr": 0.5,
"warmup_steps": 40,
"warmdown_ratio": 0.65,
"final_lr_frac": 0.05,
"resume_from_step": -1,
"init_from_checkpoint_dir": null,
"init_from_step": -1,
"no_init_optimizer": false,
"branch_lr_schedule": "branch",
"branch_parent_experiment_id": null,
"pretokenized": true,
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
"experiment_id": "Think.Unbounded-d32",
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
"tokenizer_fingerprint": "21e99d5cdeeaa660",
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
"seed": 42,
"eval_every": 250,
"eval_tokens": 2097152,
"core_metric_every": -1,
"core_metric_max_per_task": 500,
"sample_every": -1,
"save_every": 500,
"model_tag": "Think.Unbounded-d32",
"experiment": {
"schema_version": 1,
"stage": "base",
"experiment_id": "Think.Unbounded-d32",
"datasets": {
"original": {
"adapter": "parquet_shards",
"repo": "jbduran/think-dataset-clean-1930s",
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
"validation_shard": 472,
"num_train_shards": 330,
"download_workers": 4
},
"midtrain_r30": {
"adapter": "parquet_shards",
"repo": "zachnorton03/think-midtrain",
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
"subfolder": "mixed/ratio_30/data",
"validation_shard": 1262,
"num_train_shards": 391,
"download_workers": 4
},
"midtrain_r60": {
"adapter": "parquet_shards",
"repo": "zachnorton03/think-midtrain",
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
"subfolder": "mixed/ratio_60/data",
"validation_shard": 523,
"num_train_shards": 207,
"download_workers": 4
}
},
"mixture_schedule": {
"total_tokens": 20132659200,
"seed_data": 1234,
"max_epochs": 4,
"stages": [
{
"name": "base",
"start_tokens": 0,
"source": "original"
},
{
"name": "injection",
"start_tokens": 14092861440,
"source": "midtrain_r30"
},
{
"name": "decay_mix",
"start_tokens": 18119393280,
"source": "midtrain_r60"
}
]
},
"tokenizer": {
"mode": "reuse",
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
"vocab_size": 32768
},
"pretokenize": {
"enabled": true,
"slack": 1.03,
"require_no_wrap": true,
"val_tokens": 20971520,
"shard_tokens": 100000000,
"tokenizer_threads": 8
},
"training": {
"depth": 32,
"scaling_params": 1677724672,
"target_param_data_ratio": 12,
"max_seq_len": 4096,
"window_pattern": "SSSL",
"muon_momentum": 0.9,
"device_batch_size": 2,
"total_batch_size": 2097152,
"fp8": true,
"fp8_recipe": "tensorwise",
"seed": 42,
"save_every": 500,
"eval_every": 250,
"eval_tokens": 2097152,
"core_metric_every": -1,
"sample_every": -1
},
"artifacts": {
"repo": "jbduran/think.nano",
"keep_local_checkpoints": 1
},
"wandb": {
"entity": "jbduran-thinkingmachinesncsu",
"project": "think.nano",
"name": "Think.Unbounded-d32",
"group": "clean1930s-d32",
"tags": [
"think-dataset-clean-1930s",
"d32",
"ratio12",
"ctx4096",
"sssl",
"fp8",
"muon-momentum-constant",
"muon-momentum-0.90",
"midtrain-schedule",
"mix-og",
"0-30-60"
]
},
"config_fingerprint": "6f8ae461e0eb9ee2",
"artifact_path": "experiments/Think.Unbounded-d32"
},
"stage": "base",
"base_experiment_id": "Think.Unbounded-d32",
"parent_experiment_id": null,
"parent_checkpoint_step": null,
"config_fingerprint": "6f8ae461e0eb9ee2"
},
"device_batch_size": 2,
"max_seq_len": 4096,
"total_batch_size": 2097152,
"dataloader_state_dict": {
"file_idx": 41,
"pos": 94824193,
"epoch": 1,
"pq_idx": 41,
"rg_idx": 94824193,
"mixture": {
"cursors": {
"original": {
"file_idx": 41,
"pos": 94824193,
"epoch": 1,
"pq_idx": 41,
"rg_idx": 94824193
},
"midtrain_r30": {
"file_idx": 0,
"pos": 0,
"epoch": 1,
"pq_idx": 0,
"rg_idx": 0
},
"midtrain_r60": {
"file_idx": 0,
"pos": 0,
"epoch": 1,
"pq_idx": 0,
"rg_idx": 0
}
},
"cumulative_tokens": 4194312192,
"source_tokens": {
"original": 4194312192,
"midtrain_r30": 0,
"midtrain_r60": 0
},
"active_stage_idx": 0
}
},
"loop_state": {
"min_val_bpb": 0.9928518912323213,
"smooth_train_loss": 2.715263738468478,
"total_training_time": 79915.32717370987,
"stage_start_step": 0,
"stage_training_flops": 48132298326933504000,
"inherited_parent_flops": 0.0,
"cumulative_pipeline_training_flops": 4.81322983269335e+19
}
} |