Import experiment archive from bart (batch 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/summary.json +25 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/eval_metrics.json +31 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/meta_000042.json +178 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/model_000042.pt +3 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/optim_000042_rank0.pt +3 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json +95 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/evals/chatcore.json +29 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/run.json +11 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/config.json +97 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/run.json +11 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/config.json +97 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/run.json +11 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/meta_000041.json +172 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/model_000041.pt +3 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/optim_000041_rank0.pt +3 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json +89 -0
- experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/run.json +11 -0
- experiments/Think.Unbounded-d32-v2mix-cont/summary.json +79 -0
- experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/experiment_tokenizer.json +18 -0
- experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/token_bytes.pt +3 -0
- experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/tokenizer.pkl +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_000500.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_001000.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_001500.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_002000.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_002500.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_003000.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_003500.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_004000.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_004500.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_005000.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/meta_005500.json +236 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_000500.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_001000.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_001500.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_002000.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_002500.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_003000.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_003500.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_004000.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_004500.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_005000.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/model_005500.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_000500_rank0.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_001000_rank0.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_001500_rank0.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_002000_rank0.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_002500_rank0.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_003000_rank0.pt +3 -0
- experiments/Think.Unbounded-d32/base_checkpoints/optim_003500_rank0.pt +3 -0
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/summary.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1",
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 5 |
+
"parent_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 6 |
+
"parent_checkpoint_step": 9600,
|
| 7 |
+
"step": 160,
|
| 8 |
+
"stage_training_flops": 3.8505838661546803e+18,
|
| 9 |
+
"inherited_parent_flops": 2.3103503196928082e+20,
|
| 10 |
+
"cumulative_pipeline_training_flops": 2.348856158354355e+20,
|
| 11 |
+
"chatcore_metric": 0.13532147122389065,
|
| 12 |
+
"chat_results": {
|
| 13 |
+
"ARC-Easy": 0.4553872053872054,
|
| 14 |
+
"ARC-Challenge": 0.378839590443686,
|
| 15 |
+
"MMLU": 0.3474576271186441,
|
| 16 |
+
"GSM8K": 0.06444275966641395,
|
| 17 |
+
"HumanEval": 0.036585365853658534
|
| 18 |
+
},
|
| 19 |
+
"training_time_seconds": 6951.126858472824,
|
| 20 |
+
"config_fingerprint": "6359be7150ece075",
|
| 21 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 22 |
+
"git_commit_sha": "3c34775d260a016a79ab46ea76f8c8e48458fd68",
|
| 23 |
+
"wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/8c553682",
|
| 24 |
+
"huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1"
|
| 25 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/eval_metrics.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
|
| 3 |
+
"recipe": "curriculum",
|
| 4 |
+
"step": 42,
|
| 5 |
+
"val_bpb": 0.7037352212342954,
|
| 6 |
+
"min_val_bpb": 0.7037352212342954,
|
| 7 |
+
"per_route_bpb": {},
|
| 8 |
+
"per_domain_bpb": {},
|
| 9 |
+
"curriculum_summary": null,
|
| 10 |
+
"chatcore": {
|
| 11 |
+
"chatcore_metric": -0.011321608241655831,
|
| 12 |
+
"chatcore_cat": -0.018869347069426386,
|
| 13 |
+
"suite": {
|
| 14 |
+
"name": "karpathy",
|
| 15 |
+
"tasks": [
|
| 16 |
+
"ARC-Easy",
|
| 17 |
+
"ARC-Challenge",
|
| 18 |
+
"MMLU",
|
| 19 |
+
"GSM8K",
|
| 20 |
+
"HumanEval"
|
| 21 |
+
],
|
| 22 |
+
"max_generative_problems": null,
|
| 23 |
+
"generative_answer_format": null
|
| 24 |
+
},
|
| 25 |
+
"ARC-Easy": 0.24579124579124578,
|
| 26 |
+
"ARC-Challenge": 0.23037542662116042,
|
| 27 |
+
"MMLU": 0.23137729668138443,
|
| 28 |
+
"GSM8K": 0.0,
|
| 29 |
+
"HumanEval": 0.0
|
| 30 |
+
}
|
| 31 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/meta_000042.json
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 42,
|
| 3 |
+
"training_complete": true,
|
| 4 |
+
"val_bpb": 0.7037352212342954,
|
| 5 |
+
"model_config": {
|
| 6 |
+
"sequence_len": 4096,
|
| 7 |
+
"vocab_size": 32768,
|
| 8 |
+
"n_layer": 32,
|
| 9 |
+
"n_head": 16,
|
| 10 |
+
"n_kv_head": 16,
|
| 11 |
+
"n_embd": 2048,
|
| 12 |
+
"window_pattern": "SSSL"
|
| 13 |
+
},
|
| 14 |
+
"user_config": {
|
| 15 |
+
"run": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
|
| 16 |
+
"wandb_run_id": "036b1c71",
|
| 17 |
+
"wandb_group": "think-d32",
|
| 18 |
+
"wandb_tags": "sft,curriculum,c3,scale-max,staged,robustness,noise,d32",
|
| 19 |
+
"device_type": "",
|
| 20 |
+
"model_tag": null,
|
| 21 |
+
"model_step": null,
|
| 22 |
+
"base_checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/base_checkpoints",
|
| 23 |
+
"base_step": 9600,
|
| 24 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints",
|
| 25 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer",
|
| 26 |
+
"resume_from_step": null,
|
| 27 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
|
| 28 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json",
|
| 29 |
+
"parent_cumulative_flops": 2.3103503196928082e+20,
|
| 30 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 31 |
+
"git_commit_sha": "04bb043ad61d4db3e9022c422302df7b8f2dc0c9",
|
| 32 |
+
"load_optimizer": 0,
|
| 33 |
+
"num_iterations": -1,
|
| 34 |
+
"max_seq_len": null,
|
| 35 |
+
"device_batch_size": 2,
|
| 36 |
+
"total_batch_size": null,
|
| 37 |
+
"embedding_lr": null,
|
| 38 |
+
"unembedding_lr": null,
|
| 39 |
+
"matrix_lr": null,
|
| 40 |
+
"init_lr_frac": 0.8,
|
| 41 |
+
"warmup_ratio": 0.03,
|
| 42 |
+
"warmdown_ratio": 0.5,
|
| 43 |
+
"final_lr_frac": 0.0,
|
| 44 |
+
"eval_every": 400,
|
| 45 |
+
"eval_tokens": 20971520,
|
| 46 |
+
"chatcore_every": 100000,
|
| 47 |
+
"chatcore_max_cat": -1,
|
| 48 |
+
"chatcore_max_sample": 32,
|
| 49 |
+
"save_every": -1,
|
| 50 |
+
"recipe": "curriculum",
|
| 51 |
+
"curriculum_config": "",
|
| 52 |
+
"pre1930_epochs": 5,
|
| 53 |
+
"mmlu_epochs": 3,
|
| 54 |
+
"gsm8k_epochs": 4,
|
| 55 |
+
"authentic_epochs": 0,
|
| 56 |
+
"knowledge_qa_epochs": 0,
|
| 57 |
+
"multiturn_qa_epochs": 0,
|
| 58 |
+
"reasoning_qa_epochs": 0,
|
| 59 |
+
"stem_reasoning_epochs": 0,
|
| 60 |
+
"narrative_grounded_epochs": 0,
|
| 61 |
+
"narrative_fiction_epochs": 0,
|
| 62 |
+
"opinion_qa_epochs": 0,
|
| 63 |
+
"how_to_qa_epochs": 0,
|
| 64 |
+
"verse_qa_epochs": 0,
|
| 65 |
+
"composition_qa_epochs": 0,
|
| 66 |
+
"resolved_experiment_config": {
|
| 67 |
+
"schema_version": 1,
|
| 68 |
+
"stage": "sft",
|
| 69 |
+
"experiment_suffix": "pre1930-curriculum-c3-robust-v2",
|
| 70 |
+
"data": {
|
| 71 |
+
"recipe": "curriculum",
|
| 72 |
+
"curriculum": {
|
| 73 |
+
"name": "C3Rv2",
|
| 74 |
+
"mode": "staged",
|
| 75 |
+
"threshold_default": 80,
|
| 76 |
+
"stages": [
|
| 77 |
+
{
|
| 78 |
+
"routes": [
|
| 79 |
+
"knowledge_qa"
|
| 80 |
+
],
|
| 81 |
+
"authentic": "single"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"routes": [
|
| 85 |
+
"reasoning_qa",
|
| 86 |
+
"stem_reasoning",
|
| 87 |
+
"how_to_qa",
|
| 88 |
+
"opinion_qa",
|
| 89 |
+
"composition_qa",
|
| 90 |
+
"verse_qa"
|
| 91 |
+
],
|
| 92 |
+
"calibration_qa": true
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"routes": [
|
| 96 |
+
"multiturn_qa",
|
| 97 |
+
"narrative_grounded",
|
| 98 |
+
"narrative_fiction"
|
| 99 |
+
],
|
| 100 |
+
"authentic": "multi"
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"noise": {
|
| 104 |
+
"rate": 0.3
|
| 105 |
+
},
|
| 106 |
+
"robustness": {
|
| 107 |
+
"stage": 0,
|
| 108 |
+
"epochs": 1,
|
| 109 |
+
"routes": {
|
| 110 |
+
"conversation_qa": {
|
| 111 |
+
"count": null
|
| 112 |
+
},
|
| 113 |
+
"unparseable_qa": {
|
| 114 |
+
"count": null
|
| 115 |
+
},
|
| 116 |
+
"typo_qa": {
|
| 117 |
+
"count": null
|
| 118 |
+
},
|
| 119 |
+
"era_qa": {
|
| 120 |
+
"count": null
|
| 121 |
+
},
|
| 122 |
+
"conversation_multiturn": {
|
| 123 |
+
"count": null
|
| 124 |
+
}
|
| 125 |
+
}
|
| 126 |
+
}
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
"training": {
|
| 130 |
+
"num_iterations": -1,
|
| 131 |
+
"load_optimizer": 0,
|
| 132 |
+
"device_batch_size": 2,
|
| 133 |
+
"warmup_ratio": 0.03,
|
| 134 |
+
"eval_every": 400,
|
| 135 |
+
"chatcore_every": 100000,
|
| 136 |
+
"chatcore_max_sample": 32,
|
| 137 |
+
"save_every": -1
|
| 138 |
+
},
|
| 139 |
+
"artifacts": {
|
| 140 |
+
"repo": "jbduran/think.nano"
|
| 141 |
+
},
|
| 142 |
+
"wandb": {
|
| 143 |
+
"enabled": true,
|
| 144 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 145 |
+
"project": "think.nano",
|
| 146 |
+
"group": "think-d32",
|
| 147 |
+
"tags": [
|
| 148 |
+
"sft",
|
| 149 |
+
"curriculum",
|
| 150 |
+
"c3",
|
| 151 |
+
"scale-max",
|
| 152 |
+
"staged",
|
| 153 |
+
"robustness",
|
| 154 |
+
"noise",
|
| 155 |
+
"d32"
|
| 156 |
+
]
|
| 157 |
+
},
|
| 158 |
+
"config_fingerprint": "4462a389dc98cd5d",
|
| 159 |
+
"artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2"
|
| 160 |
+
},
|
| 161 |
+
"stage": "sft",
|
| 162 |
+
"base_experiment_id": null,
|
| 163 |
+
"parent_experiment_id": null,
|
| 164 |
+
"parent_checkpoint_step": null,
|
| 165 |
+
"config_fingerprint": "4462a389dc98cd5d"
|
| 166 |
+
},
|
| 167 |
+
"loop_state": {
|
| 168 |
+
"step": 42,
|
| 169 |
+
"total_training_time": 1503.5729036331177,
|
| 170 |
+
"min_val_bpb": 0.7037352212342954,
|
| 171 |
+
"smooth_train_loss": 1.8593226867746961,
|
| 172 |
+
"mfu": 51.82901425419851,
|
| 173 |
+
"tok_per_sec": 44667,
|
| 174 |
+
"stage_training_flops": 1.0107782648656036e+18,
|
| 175 |
+
"inherited_parent_flops": 2.3103503196928082e+20,
|
| 176 |
+
"cumulative_pipeline_training_flops": 2.3204581023414642e+20
|
| 177 |
+
}
|
| 178 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/model_000042.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f6a9bc83e3853f0a7ad8b27cc45388d727cc5976837f514d61ea59f80d7f0348
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/optim_000042_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:34c0871ec1aed5ced55a463ea1b827e4ef80e286735cc42512ced11f8c579fe4
|
| 3 |
+
size 11545903753
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"experiment_suffix": "pre1930-curriculum-c3-robust-v2",
|
| 5 |
+
"data": {
|
| 6 |
+
"recipe": "curriculum",
|
| 7 |
+
"curriculum": {
|
| 8 |
+
"name": "C3Rv2",
|
| 9 |
+
"mode": "staged",
|
| 10 |
+
"threshold_default": 80,
|
| 11 |
+
"stages": [
|
| 12 |
+
{
|
| 13 |
+
"routes": [
|
| 14 |
+
"knowledge_qa"
|
| 15 |
+
],
|
| 16 |
+
"authentic": "single"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"routes": [
|
| 20 |
+
"reasoning_qa",
|
| 21 |
+
"stem_reasoning",
|
| 22 |
+
"how_to_qa",
|
| 23 |
+
"opinion_qa",
|
| 24 |
+
"composition_qa",
|
| 25 |
+
"verse_qa"
|
| 26 |
+
],
|
| 27 |
+
"calibration_qa": true
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"routes": [
|
| 31 |
+
"multiturn_qa",
|
| 32 |
+
"narrative_grounded",
|
| 33 |
+
"narrative_fiction"
|
| 34 |
+
],
|
| 35 |
+
"authentic": "multi"
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"noise": {
|
| 39 |
+
"rate": 0.3
|
| 40 |
+
},
|
| 41 |
+
"robustness": {
|
| 42 |
+
"stage": 0,
|
| 43 |
+
"epochs": 1,
|
| 44 |
+
"routes": {
|
| 45 |
+
"conversation_qa": {
|
| 46 |
+
"count": null
|
| 47 |
+
},
|
| 48 |
+
"unparseable_qa": {
|
| 49 |
+
"count": null
|
| 50 |
+
},
|
| 51 |
+
"typo_qa": {
|
| 52 |
+
"count": null
|
| 53 |
+
},
|
| 54 |
+
"era_qa": {
|
| 55 |
+
"count": null
|
| 56 |
+
},
|
| 57 |
+
"conversation_multiturn": {
|
| 58 |
+
"count": null
|
| 59 |
+
}
|
| 60 |
+
}
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
},
|
| 64 |
+
"training": {
|
| 65 |
+
"num_iterations": -1,
|
| 66 |
+
"load_optimizer": 0,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"warmup_ratio": 0.03,
|
| 69 |
+
"eval_every": 400,
|
| 70 |
+
"chatcore_every": 100000,
|
| 71 |
+
"chatcore_max_sample": 32,
|
| 72 |
+
"save_every": -1
|
| 73 |
+
},
|
| 74 |
+
"artifacts": {
|
| 75 |
+
"repo": "jbduran/think.nano"
|
| 76 |
+
},
|
| 77 |
+
"wandb": {
|
| 78 |
+
"enabled": true,
|
| 79 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 80 |
+
"project": "think.nano",
|
| 81 |
+
"group": "think-d32",
|
| 82 |
+
"tags": [
|
| 83 |
+
"sft",
|
| 84 |
+
"curriculum",
|
| 85 |
+
"c3",
|
| 86 |
+
"scale-max",
|
| 87 |
+
"staged",
|
| 88 |
+
"robustness",
|
| 89 |
+
"noise",
|
| 90 |
+
"d32"
|
| 91 |
+
]
|
| 92 |
+
},
|
| 93 |
+
"config_fingerprint": "4462a389dc98cd5d",
|
| 94 |
+
"artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2"
|
| 95 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/evals/chatcore.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"stage": "sft",
|
| 3 |
+
"step": 42,
|
| 4 |
+
"total_training_flops": 2.3204581023414642e+20,
|
| 5 |
+
"stage_training_flops": 1.0107782648656036e+18,
|
| 6 |
+
"inherited_parent_flops": 2.3103503196928082e+20,
|
| 7 |
+
"cumulative_pipeline_training_flops": 2.3204581023414642e+20,
|
| 8 |
+
"results": {
|
| 9 |
+
"ARC-Easy": 0.24579124579124578,
|
| 10 |
+
"ARC-Challenge": 0.23037542662116042,
|
| 11 |
+
"MMLU": 0.23137729668138443,
|
| 12 |
+
"GSM8K": 0.0,
|
| 13 |
+
"HumanEval": 0.0
|
| 14 |
+
},
|
| 15 |
+
"chatcore_metric": -0.011321608241655831,
|
| 16 |
+
"chatcore_suite": {
|
| 17 |
+
"name": "karpathy",
|
| 18 |
+
"tasks": [
|
| 19 |
+
"ARC-Easy",
|
| 20 |
+
"ARC-Challenge",
|
| 21 |
+
"MMLU",
|
| 22 |
+
"GSM8K",
|
| 23 |
+
"HumanEval"
|
| 24 |
+
],
|
| 25 |
+
"max_generative_problems": null,
|
| 26 |
+
"generative_answer_format": null
|
| 27 |
+
},
|
| 28 |
+
"complete": true
|
| 29 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/run.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 5 |
+
"parent_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 6 |
+
"parent_checkpoint_step": 9600,
|
| 7 |
+
"branch_parent_step": null,
|
| 8 |
+
"config_fingerprint": "4462a389dc98cd5d",
|
| 9 |
+
"wandb_run_id": "036b1c71",
|
| 10 |
+
"created_at": 1786715992
|
| 11 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/config.json
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"experiment_suffix": "pre1930-curriculum-c3-robust-v3-a100-40gb",
|
| 5 |
+
"data": {
|
| 6 |
+
"recipe": "curriculum",
|
| 7 |
+
"curriculum": {
|
| 8 |
+
"name": "C3Rv3",
|
| 9 |
+
"mode": "staged",
|
| 10 |
+
"passes": 3,
|
| 11 |
+
"threshold_default": 80,
|
| 12 |
+
"stages": [
|
| 13 |
+
{
|
| 14 |
+
"routes": [
|
| 15 |
+
"knowledge_qa"
|
| 16 |
+
],
|
| 17 |
+
"authentic": "single"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"routes": [
|
| 21 |
+
"reasoning_qa",
|
| 22 |
+
"stem_reasoning",
|
| 23 |
+
"how_to_qa",
|
| 24 |
+
"opinion_qa",
|
| 25 |
+
"composition_qa",
|
| 26 |
+
"verse_qa"
|
| 27 |
+
],
|
| 28 |
+
"calibration_qa": true
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"routes": [
|
| 32 |
+
"multiturn_qa",
|
| 33 |
+
"narrative_grounded",
|
| 34 |
+
"narrative_fiction"
|
| 35 |
+
],
|
| 36 |
+
"authentic": "multi"
|
| 37 |
+
}
|
| 38 |
+
],
|
| 39 |
+
"noise": {
|
| 40 |
+
"rate": 0.3,
|
| 41 |
+
"end_punct_rate": 0.05
|
| 42 |
+
},
|
| 43 |
+
"robustness": {
|
| 44 |
+
"stage": 0,
|
| 45 |
+
"epochs": 2.5,
|
| 46 |
+
"routes": {
|
| 47 |
+
"conversation_qa": {
|
| 48 |
+
"count": null
|
| 49 |
+
},
|
| 50 |
+
"unparseable_qa": {
|
| 51 |
+
"count": null
|
| 52 |
+
},
|
| 53 |
+
"typo_qa": {
|
| 54 |
+
"count": null
|
| 55 |
+
},
|
| 56 |
+
"era_qa": {
|
| 57 |
+
"count": null
|
| 58 |
+
},
|
| 59 |
+
"conversation_multiturn": {
|
| 60 |
+
"count": null
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
},
|
| 66 |
+
"training": {
|
| 67 |
+
"num_iterations": -1,
|
| 68 |
+
"load_optimizer": 0,
|
| 69 |
+
"device_batch_size": 1,
|
| 70 |
+
"warmup_ratio": 0.03,
|
| 71 |
+
"eval_every": 400,
|
| 72 |
+
"chatcore_every": 100000,
|
| 73 |
+
"chatcore_max_sample": 32,
|
| 74 |
+
"save_every": -1
|
| 75 |
+
},
|
| 76 |
+
"artifacts": {
|
| 77 |
+
"repo": "jbduran/think.nano"
|
| 78 |
+
},
|
| 79 |
+
"wandb": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 82 |
+
"project": "think.nano",
|
| 83 |
+
"group": "think-d32",
|
| 84 |
+
"tags": [
|
| 85 |
+
"sft",
|
| 86 |
+
"curriculum",
|
| 87 |
+
"c3",
|
| 88 |
+
"scale-max",
|
| 89 |
+
"staged",
|
| 90 |
+
"robustness",
|
| 91 |
+
"noise",
|
| 92 |
+
"d32"
|
| 93 |
+
]
|
| 94 |
+
},
|
| 95 |
+
"config_fingerprint": "b0d39eef6f8e427b",
|
| 96 |
+
"artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb"
|
| 97 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/run.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb",
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 5 |
+
"parent_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 6 |
+
"parent_checkpoint_step": 9600,
|
| 7 |
+
"branch_parent_step": null,
|
| 8 |
+
"config_fingerprint": "b0d39eef6f8e427b",
|
| 9 |
+
"wandb_run_id": "5667c0f5",
|
| 10 |
+
"created_at": 1787104196
|
| 11 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/config.json
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"experiment_suffix": "pre1930-curriculum-c3-robust-v3",
|
| 5 |
+
"data": {
|
| 6 |
+
"recipe": "curriculum",
|
| 7 |
+
"curriculum": {
|
| 8 |
+
"name": "C3Rv3",
|
| 9 |
+
"mode": "staged",
|
| 10 |
+
"passes": 3,
|
| 11 |
+
"threshold_default": 80,
|
| 12 |
+
"stages": [
|
| 13 |
+
{
|
| 14 |
+
"routes": [
|
| 15 |
+
"knowledge_qa"
|
| 16 |
+
],
|
| 17 |
+
"authentic": "single"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"routes": [
|
| 21 |
+
"reasoning_qa",
|
| 22 |
+
"stem_reasoning",
|
| 23 |
+
"how_to_qa",
|
| 24 |
+
"opinion_qa",
|
| 25 |
+
"composition_qa",
|
| 26 |
+
"verse_qa"
|
| 27 |
+
],
|
| 28 |
+
"calibration_qa": true
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"routes": [
|
| 32 |
+
"multiturn_qa",
|
| 33 |
+
"narrative_grounded",
|
| 34 |
+
"narrative_fiction"
|
| 35 |
+
],
|
| 36 |
+
"authentic": "multi"
|
| 37 |
+
}
|
| 38 |
+
],
|
| 39 |
+
"noise": {
|
| 40 |
+
"rate": 0.3,
|
| 41 |
+
"end_punct_rate": 0.05
|
| 42 |
+
},
|
| 43 |
+
"robustness": {
|
| 44 |
+
"stage": 0,
|
| 45 |
+
"epochs": 2.5,
|
| 46 |
+
"routes": {
|
| 47 |
+
"conversation_qa": {
|
| 48 |
+
"count": null
|
| 49 |
+
},
|
| 50 |
+
"unparseable_qa": {
|
| 51 |
+
"count": null
|
| 52 |
+
},
|
| 53 |
+
"typo_qa": {
|
| 54 |
+
"count": null
|
| 55 |
+
},
|
| 56 |
+
"era_qa": {
|
| 57 |
+
"count": null
|
| 58 |
+
},
|
| 59 |
+
"conversation_multiturn": {
|
| 60 |
+
"count": null
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
},
|
| 66 |
+
"training": {
|
| 67 |
+
"num_iterations": -1,
|
| 68 |
+
"load_optimizer": 0,
|
| 69 |
+
"device_batch_size": 2,
|
| 70 |
+
"warmup_ratio": 0.03,
|
| 71 |
+
"eval_every": 400,
|
| 72 |
+
"chatcore_every": 100000,
|
| 73 |
+
"chatcore_max_sample": 32,
|
| 74 |
+
"save_every": -1
|
| 75 |
+
},
|
| 76 |
+
"artifacts": {
|
| 77 |
+
"repo": "jbduran/think.nano"
|
| 78 |
+
},
|
| 79 |
+
"wandb": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 82 |
+
"project": "think.nano",
|
| 83 |
+
"group": "think-d32",
|
| 84 |
+
"tags": [
|
| 85 |
+
"sft",
|
| 86 |
+
"curriculum",
|
| 87 |
+
"c3",
|
| 88 |
+
"scale-max",
|
| 89 |
+
"staged",
|
| 90 |
+
"robustness",
|
| 91 |
+
"noise",
|
| 92 |
+
"d32"
|
| 93 |
+
]
|
| 94 |
+
},
|
| 95 |
+
"config_fingerprint": "12ffd193bdca635a",
|
| 96 |
+
"artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3"
|
| 97 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/run.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3",
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 5 |
+
"parent_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 6 |
+
"parent_checkpoint_step": 9600,
|
| 7 |
+
"branch_parent_step": null,
|
| 8 |
+
"config_fingerprint": "12ffd193bdca635a",
|
| 9 |
+
"wandb_run_id": "10993832",
|
| 10 |
+
"created_at": 1787101675
|
| 11 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/meta_000041.json
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 41,
|
| 3 |
+
"training_complete": true,
|
| 4 |
+
"val_bpb": 0.7046443464143689,
|
| 5 |
+
"model_config": {
|
| 6 |
+
"sequence_len": 4096,
|
| 7 |
+
"vocab_size": 32768,
|
| 8 |
+
"n_layer": 32,
|
| 9 |
+
"n_head": 16,
|
| 10 |
+
"n_kv_head": 16,
|
| 11 |
+
"n_embd": 2048,
|
| 12 |
+
"window_pattern": "SSSL"
|
| 13 |
+
},
|
| 14 |
+
"user_config": {
|
| 15 |
+
"run": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust",
|
| 16 |
+
"wandb_run_id": "21793571",
|
| 17 |
+
"wandb_group": "think-d32",
|
| 18 |
+
"wandb_tags": "sft,curriculum,c3,scale-max,staged,robustness,noise,d32",
|
| 19 |
+
"device_type": "",
|
| 20 |
+
"model_tag": null,
|
| 21 |
+
"model_step": null,
|
| 22 |
+
"base_checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/base_checkpoints",
|
| 23 |
+
"base_step": 9600,
|
| 24 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints",
|
| 25 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer",
|
| 26 |
+
"resume_from_step": null,
|
| 27 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust",
|
| 28 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json",
|
| 29 |
+
"parent_cumulative_flops": 2.3103503196928082e+20,
|
| 30 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 31 |
+
"git_commit_sha": "8549a37b028418b0abaf347bd6e5a8ccb6cff782",
|
| 32 |
+
"load_optimizer": 0,
|
| 33 |
+
"num_iterations": -1,
|
| 34 |
+
"max_seq_len": null,
|
| 35 |
+
"device_batch_size": 2,
|
| 36 |
+
"total_batch_size": null,
|
| 37 |
+
"embedding_lr": null,
|
| 38 |
+
"unembedding_lr": null,
|
| 39 |
+
"matrix_lr": null,
|
| 40 |
+
"init_lr_frac": 0.8,
|
| 41 |
+
"warmup_ratio": 0.03,
|
| 42 |
+
"warmdown_ratio": 0.5,
|
| 43 |
+
"final_lr_frac": 0.0,
|
| 44 |
+
"eval_every": 400,
|
| 45 |
+
"eval_tokens": 20971520,
|
| 46 |
+
"chatcore_every": 100000,
|
| 47 |
+
"chatcore_max_cat": -1,
|
| 48 |
+
"chatcore_max_sample": 32,
|
| 49 |
+
"save_every": -1,
|
| 50 |
+
"recipe": "curriculum",
|
| 51 |
+
"curriculum_config": "",
|
| 52 |
+
"pre1930_epochs": 5,
|
| 53 |
+
"mmlu_epochs": 3,
|
| 54 |
+
"gsm8k_epochs": 4,
|
| 55 |
+
"authentic_epochs": 0,
|
| 56 |
+
"knowledge_qa_epochs": 0,
|
| 57 |
+
"multiturn_qa_epochs": 0,
|
| 58 |
+
"reasoning_qa_epochs": 0,
|
| 59 |
+
"stem_reasoning_epochs": 0,
|
| 60 |
+
"narrative_grounded_epochs": 0,
|
| 61 |
+
"narrative_fiction_epochs": 0,
|
| 62 |
+
"opinion_qa_epochs": 0,
|
| 63 |
+
"how_to_qa_epochs": 0,
|
| 64 |
+
"verse_qa_epochs": 0,
|
| 65 |
+
"composition_qa_epochs": 0,
|
| 66 |
+
"resolved_experiment_config": {
|
| 67 |
+
"schema_version": 1,
|
| 68 |
+
"stage": "sft",
|
| 69 |
+
"experiment_suffix": "pre1930-curriculum-c3-robust",
|
| 70 |
+
"data": {
|
| 71 |
+
"recipe": "curriculum",
|
| 72 |
+
"curriculum": {
|
| 73 |
+
"name": "C3R",
|
| 74 |
+
"mode": "staged",
|
| 75 |
+
"threshold_default": 80,
|
| 76 |
+
"stages": [
|
| 77 |
+
{
|
| 78 |
+
"routes": [
|
| 79 |
+
"knowledge_qa"
|
| 80 |
+
],
|
| 81 |
+
"authentic": "single"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"routes": [
|
| 85 |
+
"reasoning_qa",
|
| 86 |
+
"stem_reasoning",
|
| 87 |
+
"how_to_qa",
|
| 88 |
+
"opinion_qa",
|
| 89 |
+
"composition_qa",
|
| 90 |
+
"verse_qa"
|
| 91 |
+
],
|
| 92 |
+
"calibration_qa": true
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"routes": [
|
| 96 |
+
"multiturn_qa",
|
| 97 |
+
"narrative_grounded",
|
| 98 |
+
"narrative_fiction"
|
| 99 |
+
],
|
| 100 |
+
"authentic": "multi"
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"noise": {
|
| 104 |
+
"rate": 0.3
|
| 105 |
+
},
|
| 106 |
+
"robustness": {
|
| 107 |
+
"stage": 0,
|
| 108 |
+
"epochs": 1,
|
| 109 |
+
"routes": {
|
| 110 |
+
"conversation_qa": {
|
| 111 |
+
"count": null
|
| 112 |
+
},
|
| 113 |
+
"unparseable_qa": {
|
| 114 |
+
"count": null
|
| 115 |
+
},
|
| 116 |
+
"era_qa": {
|
| 117 |
+
"count": null
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
}
|
| 121 |
+
}
|
| 122 |
+
},
|
| 123 |
+
"training": {
|
| 124 |
+
"num_iterations": -1,
|
| 125 |
+
"load_optimizer": 0,
|
| 126 |
+
"device_batch_size": 2,
|
| 127 |
+
"warmup_ratio": 0.03,
|
| 128 |
+
"eval_every": 400,
|
| 129 |
+
"chatcore_every": 100000,
|
| 130 |
+
"chatcore_max_sample": 32,
|
| 131 |
+
"save_every": -1
|
| 132 |
+
},
|
| 133 |
+
"artifacts": {
|
| 134 |
+
"repo": "jbduran/think.nano"
|
| 135 |
+
},
|
| 136 |
+
"wandb": {
|
| 137 |
+
"enabled": true,
|
| 138 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 139 |
+
"project": "think.nano",
|
| 140 |
+
"group": "think-d32",
|
| 141 |
+
"tags": [
|
| 142 |
+
"sft",
|
| 143 |
+
"curriculum",
|
| 144 |
+
"c3",
|
| 145 |
+
"scale-max",
|
| 146 |
+
"staged",
|
| 147 |
+
"robustness",
|
| 148 |
+
"noise",
|
| 149 |
+
"d32"
|
| 150 |
+
]
|
| 151 |
+
},
|
| 152 |
+
"config_fingerprint": "e1a9d25fb1cf5edf",
|
| 153 |
+
"artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust"
|
| 154 |
+
},
|
| 155 |
+
"stage": "sft",
|
| 156 |
+
"base_experiment_id": null,
|
| 157 |
+
"parent_experiment_id": null,
|
| 158 |
+
"parent_checkpoint_step": null,
|
| 159 |
+
"config_fingerprint": "e1a9d25fb1cf5edf"
|
| 160 |
+
},
|
| 161 |
+
"loop_state": {
|
| 162 |
+
"step": 41,
|
| 163 |
+
"total_training_time": 1459.9074046611786,
|
| 164 |
+
"min_val_bpb": 0.7046443464143689,
|
| 165 |
+
"smooth_train_loss": 1.907550057536094,
|
| 166 |
+
"mfu": 51.698687069018604,
|
| 167 |
+
"tok_per_sec": 44555,
|
| 168 |
+
"stage_training_flops": 9.867121157021368e+17,
|
| 169 |
+
"inherited_parent_flops": 2.3103503196928082e+20,
|
| 170 |
+
"cumulative_pipeline_training_flops": 2.3202174408498296e+20
|
| 171 |
+
}
|
| 172 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/model_000041.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:590b64a7ab561ae7ec7adf10fd09469e7b46dacf76e5c665f743fd9d0dd74a6f
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/optim_000041_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7c642c3e04109fce332ca03b7afaf571812e0d9ac124f8863889f262a6748995
|
| 3 |
+
size 11545903753
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"experiment_suffix": "pre1930-curriculum-c3-robust",
|
| 5 |
+
"data": {
|
| 6 |
+
"recipe": "curriculum",
|
| 7 |
+
"curriculum": {
|
| 8 |
+
"name": "C3R",
|
| 9 |
+
"mode": "staged",
|
| 10 |
+
"threshold_default": 80,
|
| 11 |
+
"stages": [
|
| 12 |
+
{
|
| 13 |
+
"routes": [
|
| 14 |
+
"knowledge_qa"
|
| 15 |
+
],
|
| 16 |
+
"authentic": "single"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"routes": [
|
| 20 |
+
"reasoning_qa",
|
| 21 |
+
"stem_reasoning",
|
| 22 |
+
"how_to_qa",
|
| 23 |
+
"opinion_qa",
|
| 24 |
+
"composition_qa",
|
| 25 |
+
"verse_qa"
|
| 26 |
+
],
|
| 27 |
+
"calibration_qa": true
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"routes": [
|
| 31 |
+
"multiturn_qa",
|
| 32 |
+
"narrative_grounded",
|
| 33 |
+
"narrative_fiction"
|
| 34 |
+
],
|
| 35 |
+
"authentic": "multi"
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"noise": {
|
| 39 |
+
"rate": 0.3
|
| 40 |
+
},
|
| 41 |
+
"robustness": {
|
| 42 |
+
"stage": 0,
|
| 43 |
+
"epochs": 1,
|
| 44 |
+
"routes": {
|
| 45 |
+
"conversation_qa": {
|
| 46 |
+
"count": null
|
| 47 |
+
},
|
| 48 |
+
"unparseable_qa": {
|
| 49 |
+
"count": null
|
| 50 |
+
},
|
| 51 |
+
"era_qa": {
|
| 52 |
+
"count": null
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
"training": {
|
| 59 |
+
"num_iterations": -1,
|
| 60 |
+
"load_optimizer": 0,
|
| 61 |
+
"device_batch_size": 2,
|
| 62 |
+
"warmup_ratio": 0.03,
|
| 63 |
+
"eval_every": 400,
|
| 64 |
+
"chatcore_every": 100000,
|
| 65 |
+
"chatcore_max_sample": 32,
|
| 66 |
+
"save_every": -1
|
| 67 |
+
},
|
| 68 |
+
"artifacts": {
|
| 69 |
+
"repo": "jbduran/think.nano"
|
| 70 |
+
},
|
| 71 |
+
"wandb": {
|
| 72 |
+
"enabled": true,
|
| 73 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 74 |
+
"project": "think.nano",
|
| 75 |
+
"group": "think-d32",
|
| 76 |
+
"tags": [
|
| 77 |
+
"sft",
|
| 78 |
+
"curriculum",
|
| 79 |
+
"c3",
|
| 80 |
+
"scale-max",
|
| 81 |
+
"staged",
|
| 82 |
+
"robustness",
|
| 83 |
+
"noise",
|
| 84 |
+
"d32"
|
| 85 |
+
]
|
| 86 |
+
},
|
| 87 |
+
"config_fingerprint": "e1a9d25fb1cf5edf",
|
| 88 |
+
"artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust"
|
| 89 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/run.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust",
|
| 3 |
+
"stage": "sft",
|
| 4 |
+
"base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 5 |
+
"parent_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 6 |
+
"parent_checkpoint_step": 9600,
|
| 7 |
+
"branch_parent_step": null,
|
| 8 |
+
"config_fingerprint": "e1a9d25fb1cf5edf",
|
| 9 |
+
"wandb_run_id": "21793571",
|
| 10 |
+
"created_at": 1786635058
|
| 11 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/summary.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
|
| 5 |
+
"parent_experiment_id": "Think.Unbounded-d32",
|
| 6 |
+
"parent_checkpoint_step": 5500,
|
| 7 |
+
"dataset": "jbduran/think-dataset-clean-1930s",
|
| 8 |
+
"dataset_revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 9 |
+
"step": 9600,
|
| 10 |
+
"depth": 32,
|
| 11 |
+
"target_param_data_ratio": 12,
|
| 12 |
+
"training_tokens": 20132659200,
|
| 13 |
+
"final_sampled_val_bpb": 0.7708812390249374,
|
| 14 |
+
"minimum_sampled_val_bpb": 0.7701763237939314,
|
| 15 |
+
"full_val_bpb": 0.7505380497637221,
|
| 16 |
+
"core_metric": null,
|
| 17 |
+
"centered_results": null,
|
| 18 |
+
"conditioned_samples": [
|
| 19 |
+
{
|
| 20 |
+
"prompt": "The capital of France is",
|
| 21 |
+
"text": "<|bos|>The capital of France is the most important city in the world. It is the capital of the most powerful"
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"prompt": "The chemical symbol of gold is",
|
| 25 |
+
"text": "<|bos|>The chemical symbol of gold is Au, and that of silver is Ag. The symbol of the former is Ag"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"prompt": "If yesterday was Friday, then tomorrow will be",
|
| 29 |
+
"text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday was Saturday, then tomorrow will be Sunday. If yesterday was"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"prompt": "The opposite of hot is",
|
| 33 |
+
"text": "<|bos|>The opposite of hot is cold, and the opposite of cold is hot. The opposite of hot is cold"
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"prompt": "The planets of the solar system are:",
|
| 37 |
+
"text": "<|bos|>The planets of the solar system are: Mercury, Venus, Mars, Jupiter, Saturn, and Herschel. The earth"
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"prompt": "My favorite color is",
|
| 41 |
+
"text": "<|bos|>My favorite color is a deep, rich, velvety crimson, the same shade as the rose"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"prompt": "If 5*x + 3 = 13, then x is",
|
| 45 |
+
"text": "<|bos|>If 5*x + 3 = 13, then x is 5.\n\nIf 3x + 4 = 13, then x"
|
| 46 |
+
}
|
| 47 |
+
],
|
| 48 |
+
"unconditioned_samples": [
|
| 49 |
+
"<|bos|>PREFACE.\n\nTHIS essay originated with Mr. W. C. Barrett, who says of it: \"Mr. Galton's 'Life and Times,' and his Life of Darwin.' \"It is mainly a half-tone reprint from Galton's (vice supra) life, to which I have added some important facts which I had not been able to procure when the original edition was published, and which bear directly upon some matters of interest.\n\nIt may, however, be as well to draw attention more particularly to the point to which Mr. Barrett refers in the above quotation, as this contains, I believe, the fundamental distinction between the Darwin",
|
| 50 |
+
"<|bos|>Lady Saumarez -- Scotch rose shaded white. Pheasant's Eye -- Pure white, dark eye. Marigold -- Striking light golden color. Maddereti -- A most desirable purple.\n\nRose -- -A superb full flowering rose of a most beautiful shade of deep rose. Sunrise -- Pure tints of fawn and yellow.\n\nCarnation Marguerite\n\nLarge Flowered\n\nnow offered. Much hardier and blooms more abundantly.\n\nPrice, any variety, 10 cts. each; set of 1 3 for 45 cents.\n\nEasily Makes a Good Center\n\nAnywhere.\n\n",
|
| 51 |
+
"<|bos|>RIDGEN, TOLLAND\n\nFRIDLAND, Dyfe, y 1665.\n\nTHE CONFUCTIONS\n\nor\n\nTHE KINGDOM OF SOLOMON.\n\nTHE FIRST BOOK OF THE KING OF SOLOMON\n\nRefers to the Four Series of the Four Dispensations, God revealing himself to His Children in the various Births of His Israel. The Messiah's birth and baptism of John, the mission of the\n\nApostles, and the baptism of Christ, and latterly the majestic progress of His power, are chronicled in the Books of Moses, of the Prophets, of the Proph ets, and the",
|
| 52 |
+
"<|bos|>agers have already told us, and M. Crookes has already informed us, that he found concretions of this nature in the plantlice when giving a supplementary examination to them : hence, even according to his own showing, no safe conclusion can be drawn from the relative order and distribution of these silica-precipitates. At the same time it is natural to ask whether this tardy means of elucidation, instead of becoming antiquated by the lapse of the fair period which Hibbert has now completed-and well-nigh brought to a close by a misuse of some words, may not be pressed into use - --\n\nON",
|
| 53 |
+
"<|bos|>D\n\nio\n\nLe\n\nthered in the middle sept i post- ater bear ried ne pro avera hing tenanted\n\nCrowne he se ouch h find\n\n\u200bLa thi seeks depone \u0f51 \u0f4f \u0f63\u0f74 \u0f49\u0f7a\u0f51 \u0f61 \u0f61 \u0f61\u0f72 \u0f64\u0f74 \u0f62 \u0f61 \u0f51 \u0f62 \u3002 \u0f61\u0f72\u0970\u0f62 \u0f51 \u0f0b \u0f62\u0f66 Atth Id\n\nDedication of the Picture in the Library of Trinity that has the most frequently called forth criticism\n\nI am unable to",
|
| 54 |
+
"<|bos|>PREFACE\n\nCONTENTS.\n\n1. X. About a month was being wasted at the hotel, when a kindhearted friend offered to go on ahead and engage rooms for us at the best houses the day was hot-the Southern hotels always are-on the other side, and on most of them, indeed, fair profit can be made; and the diligence kept always waiting at the station and often not in readiness for an hour after the last customer had been put in, and the engineer grumbled fearfully at this irregularity of the train, because the trains gave him a pretty lively time when there was anybody waiting there to see, and the porters sometimes began",
|
| 55 |
+
"<|bos|>Clarke Model\n\n222000908000 DIANER DER\n\nThe proper unture dishes\n\nWilder Damases7",
|
| 56 |
+
"<|bos|>ossoer 0 020 oO 000000 ee OP atte eT es ee eee en eee Bee Se SERRE Nee errre reser Res eererr ee eee cerr reeer era Nee essen re ers 2008 OO OO DO ES OO ESE HD TTI MT SPEC IC PRT APOLOC H OCDOETTES 50 Ce a OC ee DDg ee CF EIOLUS) oe eA tt } 500 een eeehey n"
|
| 57 |
+
],
|
| 58 |
+
"training_time_seconds": 164042.2579112053,
|
| 59 |
+
"stage_training_flops": 9.867121157021368e+19,
|
| 60 |
+
"inherited_parent_flops": 1.3236382039906714e+20,
|
| 61 |
+
"cumulative_pipeline_training_flops": 2.3103503196928082e+20,
|
| 62 |
+
"config_fingerprint": "00af0bde578c1269",
|
| 63 |
+
"git_commit_sha": "734396fce10c8fcd7bf979261d41ebe37968089c",
|
| 64 |
+
"wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/2fa1b744",
|
| 65 |
+
"huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/Think.Unbounded-d32-v2mix-cont",
|
| 66 |
+
"dataset_fingerprint": "54cc47b13796e411",
|
| 67 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 68 |
+
"unique_train_tokens_per_source": {
|
| 69 |
+
"original": 14515647284,
|
| 70 |
+
"midtrain_r21": 4107062477,
|
| 71 |
+
"midtrain_r45": 2053531239
|
| 72 |
+
},
|
| 73 |
+
"branch_parent_experiment_id": "Think.Unbounded-d32",
|
| 74 |
+
"branch_parent_step": 5500,
|
| 75 |
+
"branch_lr_schedule": "continue",
|
| 76 |
+
"stage_training_tokens": 8598323200,
|
| 77 |
+
"unique_train_tokens": 20676241000,
|
| 78 |
+
"effective_epochs": 0.41585524177242855
|
| 79 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/experiment_tokenizer.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 3 |
+
"dataset": {
|
| 4 |
+
"adapter": "parquet_shards",
|
| 5 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 6 |
+
"revision": "main",
|
| 7 |
+
"validation_shard": 472,
|
| 8 |
+
"num_train_shards": 200,
|
| 9 |
+
"download_workers": 4
|
| 10 |
+
},
|
| 11 |
+
"tokenizer": {
|
| 12 |
+
"mode": "train",
|
| 13 |
+
"max_chars": 1000000000000,
|
| 14 |
+
"doc_cap": 1000000000,
|
| 15 |
+
"vocab_size": 32768
|
| 16 |
+
},
|
| 17 |
+
"created_at": 1784129374
|
| 18 |
+
}
|
experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/token_bytes.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2
|
| 3 |
+
size 132649
|
experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/tokenizer.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f
|
| 3 |
+
size 410542
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_000500.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 1.1476923999299866,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 10,
|
| 190 |
+
"pos": 48712193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 10,
|
| 193 |
+
"rg_idx": 48712193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 10,
|
| 198 |
+
"pos": 48712193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 10,
|
| 201 |
+
"rg_idx": 48712193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 1048584192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 1048584192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 1.1476923999299866,
|
| 229 |
+
"smooth_train_loss": 3.427469208772481,
|
| 230 |
+
"total_training_time": 19713.592538118362,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 12033074581733376000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 1.2033074581733376e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_001000.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 1.056581441012716,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 20,
|
| 190 |
+
"pos": 97416193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 20,
|
| 193 |
+
"rg_idx": 97416193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 20,
|
| 198 |
+
"pos": 97416193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 20,
|
| 201 |
+
"rg_idx": 97416193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 2097160192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 2097160192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 1.056581441012716,
|
| 229 |
+
"smooth_train_loss": 2.9296221567382985,
|
| 230 |
+
"total_training_time": 39794.540657281876,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 24066149163466752000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 2.406614916346675e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_001500.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 1.0152464516907762,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 31,
|
| 190 |
+
"pos": 46120193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 31,
|
| 193 |
+
"rg_idx": 46120193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 31,
|
| 198 |
+
"pos": 46120193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 31,
|
| 201 |
+
"rg_idx": 46120193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 3145736192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 3145736192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 1.0152464516907762,
|
| 229 |
+
"smooth_train_loss": 2.935989382480182,
|
| 230 |
+
"total_training_time": 59860.37234377861,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 36099223745200128000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 3.609922374520013e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_002000.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.9928518912323213,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 41,
|
| 190 |
+
"pos": 94824193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 41,
|
| 193 |
+
"rg_idx": 94824193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 41,
|
| 198 |
+
"pos": 94824193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 41,
|
| 201 |
+
"rg_idx": 94824193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 4194312192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 4194312192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.9928518912323213,
|
| 229 |
+
"smooth_train_loss": 2.715263738468478,
|
| 230 |
+
"total_training_time": 79915.32717370987,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 48132298326933504000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 4.81322983269335e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_002500.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.9737910804428661,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 52,
|
| 190 |
+
"pos": 43528193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 52,
|
| 193 |
+
"rg_idx": 43528193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 52,
|
| 198 |
+
"pos": 43528193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 52,
|
| 201 |
+
"rg_idx": 43528193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 5242888192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 5242888192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.9630578341851044,
|
| 229 |
+
"smooth_train_loss": 2.7360114404103504,
|
| 230 |
+
"total_training_time": 99964.06782126427,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 60165372908666880000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 6.016537290866688e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_003000.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 3000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.9735215125930343,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 62,
|
| 190 |
+
"pos": 92232193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 62,
|
| 193 |
+
"rg_idx": 92232193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 62,
|
| 198 |
+
"pos": 92232193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 62,
|
| 201 |
+
"rg_idx": 92232193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 6291464192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 6291464192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.9630578341851044,
|
| 229 |
+
"smooth_train_loss": 2.6967057059495345,
|
| 230 |
+
"total_training_time": 120021.50824666023,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 72198447490400256000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 7.219844749040026e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_003500.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 3500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.968207957699399,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 73,
|
| 190 |
+
"pos": 40936193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 73,
|
| 193 |
+
"rg_idx": 40936193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 73,
|
| 198 |
+
"pos": 40936193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 73,
|
| 201 |
+
"rg_idx": 40936193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 7340040192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 7340040192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.9630578341851044,
|
| 229 |
+
"smooth_train_loss": 2.5911028486222323,
|
| 230 |
+
"total_training_time": 140049.3800020218,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 84231522072133632000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 8.423152207213363e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_004000.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 4000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.9585141600611977,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 83,
|
| 190 |
+
"pos": 89640193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 83,
|
| 193 |
+
"rg_idx": 89640193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 83,
|
| 198 |
+
"pos": 89640193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 83,
|
| 201 |
+
"rg_idx": 89640193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 8388616192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 8388616192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.9585141600611977,
|
| 229 |
+
"smooth_train_loss": 2.7874954028564223,
|
| 230 |
+
"total_training_time": 160072.53116488457,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 96264596653867008000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 9.6264596653867e+19
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_004500.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 4500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.956614818178498,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 94,
|
| 190 |
+
"pos": 38344193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 94,
|
| 193 |
+
"rg_idx": 38344193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 94,
|
| 198 |
+
"pos": 38344193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 94,
|
| 201 |
+
"rg_idx": 38344193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 9437192192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 9437192192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.956614818178498,
|
| 229 |
+
"smooth_train_loss": 2.591393936107061,
|
| 230 |
+
"total_training_time": 180101.40045905113,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 108297671235600384000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 1.0829767123560038e+20
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_005000.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 5000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.9363505794880082,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 104,
|
| 190 |
+
"pos": 87048193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 104,
|
| 193 |
+
"rg_idx": 87048193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 104,
|
| 198 |
+
"pos": 87048193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 104,
|
| 201 |
+
"rg_idx": 87048193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 10485768192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 10485768192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.9363505794880082,
|
| 229 |
+
"smooth_train_loss": 2.4582942112751494,
|
| 230 |
+
"total_training_time": 200133.8836593628,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 120330745817333760000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 1.2033074581733376e+20
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/meta_005500.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 5500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"val_bpb": 0.8999200969809518,
|
| 8 |
+
"model_config": {
|
| 9 |
+
"sequence_len": 4096,
|
| 10 |
+
"vocab_size": 32768,
|
| 11 |
+
"n_layer": 32,
|
| 12 |
+
"n_head": 16,
|
| 13 |
+
"n_kv_head": 16,
|
| 14 |
+
"n_embd": 2048,
|
| 15 |
+
"window_pattern": "SSSL"
|
| 16 |
+
},
|
| 17 |
+
"user_config": {
|
| 18 |
+
"run": "Think.Unbounded-d32",
|
| 19 |
+
"wandb_run_id": "9b75691b",
|
| 20 |
+
"wandb_group": "clean1930s-d32",
|
| 21 |
+
"wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60",
|
| 22 |
+
"device_type": "",
|
| 23 |
+
"fp8": true,
|
| 24 |
+
"fp8_recipe": "tensorwise",
|
| 25 |
+
"depth": 32,
|
| 26 |
+
"aspect_ratio": 64,
|
| 27 |
+
"head_dim": 128,
|
| 28 |
+
"max_seq_len": 4096,
|
| 29 |
+
"window_pattern": "SSSL",
|
| 30 |
+
"num_iterations": 9600,
|
| 31 |
+
"target_flops": -1.0,
|
| 32 |
+
"target_param_data_ratio": -1.0,
|
| 33 |
+
"device_batch_size": 2,
|
| 34 |
+
"total_batch_size": 2097152,
|
| 35 |
+
"embedding_lr": 0.3,
|
| 36 |
+
"unembedding_lr": 0.008,
|
| 37 |
+
"weight_decay": 0.28,
|
| 38 |
+
"matrix_lr": 0.02,
|
| 39 |
+
"muon_momentum": 0.9,
|
| 40 |
+
"scalar_lr": 0.5,
|
| 41 |
+
"warmup_steps": 40,
|
| 42 |
+
"warmdown_ratio": 0.65,
|
| 43 |
+
"final_lr_frac": 0.05,
|
| 44 |
+
"resume_from_step": -1,
|
| 45 |
+
"init_from_checkpoint_dir": null,
|
| 46 |
+
"init_from_step": -1,
|
| 47 |
+
"no_init_optimizer": false,
|
| 48 |
+
"branch_lr_schedule": "branch",
|
| 49 |
+
"branch_parent_experiment_id": null,
|
| 50 |
+
"pretokenized": true,
|
| 51 |
+
"data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data",
|
| 52 |
+
"tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer",
|
| 53 |
+
"pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok",
|
| 54 |
+
"mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}",
|
| 55 |
+
"checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints",
|
| 56 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 57 |
+
"experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json",
|
| 58 |
+
"tokenizer_fingerprint": "21e99d5cdeeaa660",
|
| 59 |
+
"git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24",
|
| 60 |
+
"seed": 42,
|
| 61 |
+
"eval_every": 250,
|
| 62 |
+
"eval_tokens": 2097152,
|
| 63 |
+
"core_metric_every": -1,
|
| 64 |
+
"core_metric_max_per_task": 500,
|
| 65 |
+
"sample_every": -1,
|
| 66 |
+
"save_every": 500,
|
| 67 |
+
"model_tag": "Think.Unbounded-d32",
|
| 68 |
+
"experiment": {
|
| 69 |
+
"schema_version": 1,
|
| 70 |
+
"stage": "base",
|
| 71 |
+
"experiment_id": "Think.Unbounded-d32",
|
| 72 |
+
"datasets": {
|
| 73 |
+
"original": {
|
| 74 |
+
"adapter": "parquet_shards",
|
| 75 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 76 |
+
"revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
|
| 77 |
+
"validation_shard": 472,
|
| 78 |
+
"num_train_shards": 330,
|
| 79 |
+
"download_workers": 4
|
| 80 |
+
},
|
| 81 |
+
"midtrain_r30": {
|
| 82 |
+
"adapter": "parquet_shards",
|
| 83 |
+
"repo": "zachnorton03/think-midtrain",
|
| 84 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 85 |
+
"subfolder": "mixed/ratio_30/data",
|
| 86 |
+
"validation_shard": 1262,
|
| 87 |
+
"num_train_shards": 391,
|
| 88 |
+
"download_workers": 4
|
| 89 |
+
},
|
| 90 |
+
"midtrain_r60": {
|
| 91 |
+
"adapter": "parquet_shards",
|
| 92 |
+
"repo": "zachnorton03/think-midtrain",
|
| 93 |
+
"revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
|
| 94 |
+
"subfolder": "mixed/ratio_60/data",
|
| 95 |
+
"validation_shard": 523,
|
| 96 |
+
"num_train_shards": 207,
|
| 97 |
+
"download_workers": 4
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"mixture_schedule": {
|
| 101 |
+
"total_tokens": 20132659200,
|
| 102 |
+
"seed_data": 1234,
|
| 103 |
+
"max_epochs": 4,
|
| 104 |
+
"stages": [
|
| 105 |
+
{
|
| 106 |
+
"name": "base",
|
| 107 |
+
"start_tokens": 0,
|
| 108 |
+
"source": "original"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"name": "injection",
|
| 112 |
+
"start_tokens": 14092861440,
|
| 113 |
+
"source": "midtrain_r30"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "decay_mix",
|
| 117 |
+
"start_tokens": 18119393280,
|
| 118 |
+
"source": "midtrain_r60"
|
| 119 |
+
}
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"tokenizer": {
|
| 123 |
+
"mode": "reuse",
|
| 124 |
+
"source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
|
| 125 |
+
"vocab_size": 32768
|
| 126 |
+
},
|
| 127 |
+
"pretokenize": {
|
| 128 |
+
"enabled": true,
|
| 129 |
+
"slack": 1.03,
|
| 130 |
+
"require_no_wrap": true,
|
| 131 |
+
"val_tokens": 20971520,
|
| 132 |
+
"shard_tokens": 100000000,
|
| 133 |
+
"tokenizer_threads": 8
|
| 134 |
+
},
|
| 135 |
+
"training": {
|
| 136 |
+
"depth": 32,
|
| 137 |
+
"scaling_params": 1677724672,
|
| 138 |
+
"target_param_data_ratio": 12,
|
| 139 |
+
"max_seq_len": 4096,
|
| 140 |
+
"window_pattern": "SSSL",
|
| 141 |
+
"muon_momentum": 0.9,
|
| 142 |
+
"device_batch_size": 2,
|
| 143 |
+
"total_batch_size": 2097152,
|
| 144 |
+
"fp8": true,
|
| 145 |
+
"fp8_recipe": "tensorwise",
|
| 146 |
+
"seed": 42,
|
| 147 |
+
"save_every": 500,
|
| 148 |
+
"eval_every": 250,
|
| 149 |
+
"eval_tokens": 2097152,
|
| 150 |
+
"core_metric_every": -1,
|
| 151 |
+
"sample_every": -1
|
| 152 |
+
},
|
| 153 |
+
"artifacts": {
|
| 154 |
+
"repo": "jbduran/think.nano",
|
| 155 |
+
"keep_local_checkpoints": 1
|
| 156 |
+
},
|
| 157 |
+
"wandb": {
|
| 158 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 159 |
+
"project": "think.nano",
|
| 160 |
+
"name": "Think.Unbounded-d32",
|
| 161 |
+
"group": "clean1930s-d32",
|
| 162 |
+
"tags": [
|
| 163 |
+
"think-dataset-clean-1930s",
|
| 164 |
+
"d32",
|
| 165 |
+
"ratio12",
|
| 166 |
+
"ctx4096",
|
| 167 |
+
"sssl",
|
| 168 |
+
"fp8",
|
| 169 |
+
"muon-momentum-constant",
|
| 170 |
+
"muon-momentum-0.90",
|
| 171 |
+
"midtrain-schedule",
|
| 172 |
+
"mix-og",
|
| 173 |
+
"0-30-60"
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"config_fingerprint": "6f8ae461e0eb9ee2",
|
| 177 |
+
"artifact_path": "experiments/Think.Unbounded-d32"
|
| 178 |
+
},
|
| 179 |
+
"stage": "base",
|
| 180 |
+
"base_experiment_id": "Think.Unbounded-d32",
|
| 181 |
+
"parent_experiment_id": null,
|
| 182 |
+
"parent_checkpoint_step": null,
|
| 183 |
+
"config_fingerprint": "6f8ae461e0eb9ee2"
|
| 184 |
+
},
|
| 185 |
+
"device_batch_size": 2,
|
| 186 |
+
"max_seq_len": 4096,
|
| 187 |
+
"total_batch_size": 2097152,
|
| 188 |
+
"dataloader_state_dict": {
|
| 189 |
+
"file_idx": 115,
|
| 190 |
+
"pos": 35752193,
|
| 191 |
+
"epoch": 1,
|
| 192 |
+
"pq_idx": 115,
|
| 193 |
+
"rg_idx": 35752193,
|
| 194 |
+
"mixture": {
|
| 195 |
+
"cursors": {
|
| 196 |
+
"original": {
|
| 197 |
+
"file_idx": 115,
|
| 198 |
+
"pos": 35752193,
|
| 199 |
+
"epoch": 1,
|
| 200 |
+
"pq_idx": 115,
|
| 201 |
+
"rg_idx": 35752193
|
| 202 |
+
},
|
| 203 |
+
"midtrain_r30": {
|
| 204 |
+
"file_idx": 0,
|
| 205 |
+
"pos": 0,
|
| 206 |
+
"epoch": 1,
|
| 207 |
+
"pq_idx": 0,
|
| 208 |
+
"rg_idx": 0
|
| 209 |
+
},
|
| 210 |
+
"midtrain_r60": {
|
| 211 |
+
"file_idx": 0,
|
| 212 |
+
"pos": 0,
|
| 213 |
+
"epoch": 1,
|
| 214 |
+
"pq_idx": 0,
|
| 215 |
+
"rg_idx": 0
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"cumulative_tokens": 11534344192,
|
| 219 |
+
"source_tokens": {
|
| 220 |
+
"original": 11534344192,
|
| 221 |
+
"midtrain_r30": 0,
|
| 222 |
+
"midtrain_r60": 0
|
| 223 |
+
},
|
| 224 |
+
"active_stage_idx": 0
|
| 225 |
+
}
|
| 226 |
+
},
|
| 227 |
+
"loop_state": {
|
| 228 |
+
"min_val_bpb": 0.8999200969809518,
|
| 229 |
+
"smooth_train_loss": 2.4195481055384045,
|
| 230 |
+
"total_training_time": 220148.93892598152,
|
| 231 |
+
"stage_start_step": 0,
|
| 232 |
+
"stage_training_flops": 132363820399067136000,
|
| 233 |
+
"inherited_parent_flops": 0.0,
|
| 234 |
+
"cumulative_pipeline_training_flops": 1.3236382039906714e+20
|
| 235 |
+
}
|
| 236 |
+
}
|
experiments/Think.Unbounded-d32/base_checkpoints/model_000500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:adc847ba9a4b6768498bbf6b84212825ac86098a137a340fd2eaefa72c01a021
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2ec635fc9fd654db070738b9ebc8b84f8b1169d60682241c8836cde1b5bf8a71
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_001500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2da68abcf5dbd78f3af6b00b815a538bda1955d7977ba49697b0aec3843bab2f
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_002000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d7e811bb7d332aa76a8593587752f27efcea20af3a03a221d2ae3a7abad4aef2
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_002500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e9bc3c0f1ed3f5510185c16226614281cc7187b71ddcf53447054985be764722
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_003000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e6f2485fd307a258eb3fded1bc9b1be934ac2184fbf2eefdf10d850bba36cb13
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_003500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cc4d43cb19a8d1c085f0d3a9ee21b023ea1ee5a4f573896f628f2f169503aeaa
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_004000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d1c498d9afbfe6ef3231f42221c888f3cf89e17ed6526b59562b40b1df47475a
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_004500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9bba2104bb0b0dc658dc52e9cc4c31ec0356e0d051f30e8ddb3d1dce564008ce
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_005000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7603f520faeabb51df975a35ab8b0065e5ec14782eceb8077dca635045d05f1a
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/model_005500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e40a97ef3206dbb1c16ebcb1d587055eaee6438f057d150ba2f0df78345b372a
|
| 3 |
+
size 8992694653
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_000500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:060bd64c5f41a991f7822484d4856efc90683c9530eed9fb3ffbdf286428415f
|
| 3 |
+
size 11545903817
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_001000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b93c1121cdb0fb1a70add6e4ce189ea7d2fc79ec6b0348ecddcc5b3e8e141ac4
|
| 3 |
+
size 11545903817
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_001500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:18141073e073d931961e0c105037de5db7e46113e84f0bfd9faac16e92a8dd6e
|
| 3 |
+
size 11545903817
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_002000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fb065a47e6c313adf94f475ef87686934efc532c0e0708aa19ab1fb18778074d
|
| 3 |
+
size 11545903817
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_002500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b23c21030cd25dc859e938236993159aa0c951398c99904083e984cd8c9208f5
|
| 3 |
+
size 11545903817
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_003000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e56c1e1a3a6e4eff55d3df7eb840e1a2ed60b9e314c1a854ccdbcf2f2a498aee
|
| 3 |
+
size 11545903817
|
experiments/Think.Unbounded-d32/base_checkpoints/optim_003500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:09542bfde4246e07a20f2f250562204f2b03c8044963b00cdb30808c357e8f94
|
| 3 |
+
size 11545903817
|