| { |
| "step": 5000, |
| "training_complete": false, |
| "experiment_id": "Think.Unbounded-d32", |
| "parent_experiment_id": null, |
| "parent_checkpoint_step": null, |
| "val_bpb": 0.9363505794880082, |
| "model_config": { |
| "sequence_len": 4096, |
| "vocab_size": 32768, |
| "n_layer": 32, |
| "n_head": 16, |
| "n_kv_head": 16, |
| "n_embd": 2048, |
| "window_pattern": "SSSL" |
| }, |
| "user_config": { |
| "run": "Think.Unbounded-d32", |
| "wandb_run_id": "9b75691b", |
| "wandb_group": "clean1930s-d32", |
| "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", |
| "device_type": "", |
| "fp8": true, |
| "fp8_recipe": "tensorwise", |
| "depth": 32, |
| "aspect_ratio": 64, |
| "head_dim": 128, |
| "max_seq_len": 4096, |
| "window_pattern": "SSSL", |
| "num_iterations": 9600, |
| "target_flops": -1.0, |
| "target_param_data_ratio": -1.0, |
| "device_batch_size": 2, |
| "total_batch_size": 2097152, |
| "embedding_lr": 0.3, |
| "unembedding_lr": 0.008, |
| "weight_decay": 0.28, |
| "matrix_lr": 0.02, |
| "muon_momentum": 0.9, |
| "scalar_lr": 0.5, |
| "warmup_steps": 40, |
| "warmdown_ratio": 0.65, |
| "final_lr_frac": 0.05, |
| "resume_from_step": -1, |
| "init_from_checkpoint_dir": null, |
| "init_from_step": -1, |
| "no_init_optimizer": false, |
| "branch_lr_schedule": "branch", |
| "branch_parent_experiment_id": null, |
| "pretokenized": true, |
| "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", |
| "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", |
| "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", |
| "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", |
| "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", |
| "experiment_id": "Think.Unbounded-d32", |
| "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", |
| "tokenizer_fingerprint": "21e99d5cdeeaa660", |
| "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", |
| "seed": 42, |
| "eval_every": 250, |
| "eval_tokens": 2097152, |
| "core_metric_every": -1, |
| "core_metric_max_per_task": 500, |
| "sample_every": -1, |
| "save_every": 500, |
| "model_tag": "Think.Unbounded-d32", |
| "experiment": { |
| "schema_version": 1, |
| "stage": "base", |
| "experiment_id": "Think.Unbounded-d32", |
| "datasets": { |
| "original": { |
| "adapter": "parquet_shards", |
| "repo": "jbduran/think-dataset-clean-1930s", |
| "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", |
| "validation_shard": 472, |
| "num_train_shards": 330, |
| "download_workers": 4 |
| }, |
| "midtrain_r30": { |
| "adapter": "parquet_shards", |
| "repo": "zachnorton03/think-midtrain", |
| "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", |
| "subfolder": "mixed/ratio_30/data", |
| "validation_shard": 1262, |
| "num_train_shards": 391, |
| "download_workers": 4 |
| }, |
| "midtrain_r60": { |
| "adapter": "parquet_shards", |
| "repo": "zachnorton03/think-midtrain", |
| "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", |
| "subfolder": "mixed/ratio_60/data", |
| "validation_shard": 523, |
| "num_train_shards": 207, |
| "download_workers": 4 |
| } |
| }, |
| "mixture_schedule": { |
| "total_tokens": 20132659200, |
| "seed_data": 1234, |
| "max_epochs": 4, |
| "stages": [ |
| { |
| "name": "base", |
| "start_tokens": 0, |
| "source": "original" |
| }, |
| { |
| "name": "injection", |
| "start_tokens": 14092861440, |
| "source": "midtrain_r30" |
| }, |
| { |
| "name": "decay_mix", |
| "start_tokens": 18119393280, |
| "source": "midtrain_r60" |
| } |
| ] |
| }, |
| "tokenizer": { |
| "mode": "reuse", |
| "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", |
| "vocab_size": 32768 |
| }, |
| "pretokenize": { |
| "enabled": true, |
| "slack": 1.03, |
| "require_no_wrap": true, |
| "val_tokens": 20971520, |
| "shard_tokens": 100000000, |
| "tokenizer_threads": 8 |
| }, |
| "training": { |
| "depth": 32, |
| "scaling_params": 1677724672, |
| "target_param_data_ratio": 12, |
| "max_seq_len": 4096, |
| "window_pattern": "SSSL", |
| "muon_momentum": 0.9, |
| "device_batch_size": 2, |
| "total_batch_size": 2097152, |
| "fp8": true, |
| "fp8_recipe": "tensorwise", |
| "seed": 42, |
| "save_every": 500, |
| "eval_every": 250, |
| "eval_tokens": 2097152, |
| "core_metric_every": -1, |
| "sample_every": -1 |
| }, |
| "artifacts": { |
| "repo": "jbduran/think.nano", |
| "keep_local_checkpoints": 1 |
| }, |
| "wandb": { |
| "entity": "jbduran-thinkingmachinesncsu", |
| "project": "think.nano", |
| "name": "Think.Unbounded-d32", |
| "group": "clean1930s-d32", |
| "tags": [ |
| "think-dataset-clean-1930s", |
| "d32", |
| "ratio12", |
| "ctx4096", |
| "sssl", |
| "fp8", |
| "muon-momentum-constant", |
| "muon-momentum-0.90", |
| "midtrain-schedule", |
| "mix-og", |
| "0-30-60" |
| ] |
| }, |
| "config_fingerprint": "6f8ae461e0eb9ee2", |
| "artifact_path": "experiments/Think.Unbounded-d32" |
| }, |
| "stage": "base", |
| "base_experiment_id": "Think.Unbounded-d32", |
| "parent_experiment_id": null, |
| "parent_checkpoint_step": null, |
| "config_fingerprint": "6f8ae461e0eb9ee2" |
| }, |
| "device_batch_size": 2, |
| "max_seq_len": 4096, |
| "total_batch_size": 2097152, |
| "dataloader_state_dict": { |
| "file_idx": 104, |
| "pos": 87048193, |
| "epoch": 1, |
| "pq_idx": 104, |
| "rg_idx": 87048193, |
| "mixture": { |
| "cursors": { |
| "original": { |
| "file_idx": 104, |
| "pos": 87048193, |
| "epoch": 1, |
| "pq_idx": 104, |
| "rg_idx": 87048193 |
| }, |
| "midtrain_r30": { |
| "file_idx": 0, |
| "pos": 0, |
| "epoch": 1, |
| "pq_idx": 0, |
| "rg_idx": 0 |
| }, |
| "midtrain_r60": { |
| "file_idx": 0, |
| "pos": 0, |
| "epoch": 1, |
| "pq_idx": 0, |
| "rg_idx": 0 |
| } |
| }, |
| "cumulative_tokens": 10485768192, |
| "source_tokens": { |
| "original": 10485768192, |
| "midtrain_r30": 0, |
| "midtrain_r60": 0 |
| }, |
| "active_stage_idx": 0 |
| } |
| }, |
| "loop_state": { |
| "min_val_bpb": 0.9363505794880082, |
| "smooth_train_loss": 2.4582942112751494, |
| "total_training_time": 200133.8836593628, |
| "stage_start_step": 0, |
| "stage_training_flops": 120330745817333760000, |
| "inherited_parent_flops": 0.0, |
| "cumulative_pipeline_training_flops": 1.2033074581733376e+20 |
| } |
| } |
