| { |
| "schema_version": 1, |
| "stage": "base", |
| "experiment_id": "Think.Unbounded-d32-v2mix-cont", |
| "branch": { |
| "parent_experiment_id": "Think.Unbounded-d32", |
| "parent_step": 5500, |
| "lr_schedule": "continue", |
| "load_optimizer": true |
| }, |
| "datasets": { |
| "original": { |
| "adapter": "parquet_shards", |
| "repo": "jbduran/think-dataset-clean-1930s", |
| "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", |
| "validation_shard": 472, |
| "num_train_shards": 330, |
| "download_workers": 4 |
| }, |
| "midtrain_r21": { |
| "adapter": "parquet_shards", |
| "repo": "zachnorton03/think-midtrain", |
| "revision": "5250c497c8c72558a573dbcd216e26ea3c76e6e2", |
| "subfolder": "mixed/v2/ratio_21/data", |
| "validation_shard": 65, |
| "num_train_shards": 65, |
| "download_workers": 4 |
| }, |
| "midtrain_r45": { |
| "adapter": "parquet_shards", |
| "repo": "zachnorton03/think-midtrain", |
| "revision": "5250c497c8c72558a573dbcd216e26ea3c76e6e2", |
| "subfolder": "mixed/v2/ratio_45/data", |
| "validation_shard": 33, |
| "num_train_shards": 33, |
| "download_workers": 4 |
| } |
| }, |
| "mixture_schedule": { |
| "total_tokens": 20132659200, |
| "seed_data": 1234, |
| "max_epochs": 4, |
| "stages": [ |
| { |
| "name": "base", |
| "start_tokens": 0, |
| "source": "original" |
| }, |
| { |
| "name": "injection", |
| "start_tokens": 14092861440, |
| "source": "midtrain_r21" |
| }, |
| { |
| "name": "decay_mix", |
| "start_tokens": 18119393280, |
| "source": "midtrain_r45" |
| } |
| ] |
| }, |
| "tokenizer": { |
| "mode": "reuse", |
| "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", |
| "vocab_size": 32768 |
| }, |
| "pretokenize": { |
| "enabled": true, |
| "slack": 1.02, |
| "require_no_wrap": true, |
| "val_tokens": 20971520, |
| "shard_tokens": 100000000, |
| "tokenizer_threads": 8 |
| }, |
| "training": { |
| "depth": 32, |
| "scaling_params": 1677724672, |
| "target_param_data_ratio": 12, |
| "max_seq_len": 4096, |
| "window_pattern": "SSSL", |
| "muon_momentum": 0.9, |
| "device_batch_size": 2, |
| "total_batch_size": 2097152, |
| "fp8": true, |
| "fp8_recipe": "tensorwise", |
| "seed": 42, |
| "save_every": 500, |
| "eval_every": 250, |
| "eval_tokens": 2097152, |
| "core_metric_every": -1, |
| "sample_every": -1 |
| }, |
| "artifacts": { |
| "repo": "jbduran/think.nano", |
| "keep_local_checkpoints": 1 |
| }, |
| "wandb": { |
| "entity": "jbduran-thinkingmachinesncsu", |
| "project": "think.nano", |
| "name": "Think.Unbounded-d32-v2mix-cont", |
| "group": "clean1930s-d32", |
| "tags": [ |
| "think-dataset-clean-1930s", |
| "d32", |
| "ratio12", |
| "ctx4096", |
| "sssl", |
| "fp8", |
| "muon-momentum-constant", |
| "muon-momentum-0.90", |
| "midtrain-schedule", |
| "mix-v2", |
| "0-21-45", |
| "midtrain-2.88ep", |
| "data-repair-continuation" |
| ] |
| }, |
| "config_fingerprint": "00af0bde578c1269", |
| "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont" |
| } |
|
|
