| { |
| "schema_version": 1, |
| "stage": "base", |
| "experiment_id": "Think.Unbounded-d32", |
| "datasets": { |
| "original": { |
| "adapter": "parquet_shards", |
| "repo": "jbduran/think-dataset-clean-1930s", |
| "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", |
| "validation_shard": 472, |
| "num_train_shards": 330, |
| "download_workers": 4 |
| }, |
| "midtrain_r30": { |
| "adapter": "parquet_shards", |
| "repo": "zachnorton03/think-midtrain", |
| "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", |
| "subfolder": "mixed/ratio_30/data", |
| "validation_shard": 1262, |
| "num_train_shards": 391, |
| "download_workers": 4 |
| }, |
| "midtrain_r60": { |
| "adapter": "parquet_shards", |
| "repo": "zachnorton03/think-midtrain", |
| "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", |
| "subfolder": "mixed/ratio_60/data", |
| "validation_shard": 523, |
| "num_train_shards": 207, |
| "download_workers": 4 |
| } |
| }, |
| "mixture_schedule": { |
| "total_tokens": 20132659200, |
| "seed_data": 1234, |
| "max_epochs": 4, |
| "stages": [ |
| { |
| "name": "base", |
| "start_tokens": 0, |
| "source": "original" |
| }, |
| { |
| "name": "injection", |
| "start_tokens": 14092861440, |
| "source": "midtrain_r30" |
| }, |
| { |
| "name": "decay_mix", |
| "start_tokens": 18119393280, |
| "source": "midtrain_r60" |
| } |
| ] |
| }, |
| "tokenizer": { |
| "mode": "reuse", |
| "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", |
| "vocab_size": 32768 |
| }, |
| "pretokenize": { |
| "enabled": true, |
| "slack": 1.03, |
| "require_no_wrap": true, |
| "val_tokens": 20971520, |
| "shard_tokens": 100000000, |
| "tokenizer_threads": 8 |
| }, |
| "training": { |
| "depth": 32, |
| "scaling_params": 1677724672, |
| "target_param_data_ratio": 12, |
| "max_seq_len": 4096, |
| "window_pattern": "SSSL", |
| "muon_momentum": 0.9, |
| "device_batch_size": 2, |
| "total_batch_size": 2097152, |
| "fp8": true, |
| "fp8_recipe": "tensorwise", |
| "seed": 42, |
| "save_every": 500, |
| "eval_every": 250, |
| "eval_tokens": 2097152, |
| "core_metric_every": -1, |
| "sample_every": -1 |
| }, |
| "artifacts": { |
| "repo": "jbduran/think.nano", |
| "keep_local_checkpoints": 1 |
| }, |
| "wandb": { |
| "entity": "jbduran-thinkingmachinesncsu", |
| "project": "think.nano", |
| "name": "Think.Unbounded-d32", |
| "group": "clean1930s-d32", |
| "tags": [ |
| "think-dataset-clean-1930s", |
| "d32", |
| "ratio12", |
| "ctx4096", |
| "sssl", |
| "fp8", |
| "muon-momentum-constant", |
| "muon-momentum-0.90", |
| "midtrain-schedule", |
| "mix-og", |
| "0-30-60" |
| ] |
| }, |
| "config_fingerprint": "6f8ae461e0eb9ee2", |
| "artifact_path": "experiments/Think.Unbounded-d32" |
| } |
|
|
