Hugging Face's logo

jbduran
/
think.nano

PyTorch
nanochat
language-model
Model card Files
xet
 Community
2
jbduran commited on
Commit
753f4af
·
verified ·
1 Parent(s): 668f24c

Upload 1930s-d12-r12-4096ctx-run2 step 2520

Browse files
Files changed (1) hide show
  1. experiments/1930s-d12-r12-4096ctx-run2/base_checkpoints/meta_002520.json +224 -0
experiments/1930s-d12-r12-4096ctx-run2/base_checkpoints/meta_002520.json ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 2520,
3
+ "training_complete": true,
4
+ "experiment_id": "1930s-d12-r12-4096ctx-run2",
5
+ "val_bpb": 1.0879491465161388,
6
+ "model_config": {
7
+ "sequence_len": 4096,
8
+ "vocab_size": 32768,
9
+ "n_layer": 12,
10
+ "n_head": 6,
11
+ "n_kv_head": 6,
12
+ "n_embd": 768,
13
+ "window_pattern": "L"
14
+ },
15
+ "user_config": {
16
+ "run": "1930s-d12-r12-4096ctx-run2",
17
+ "wandb_run_id": "f20b8b2f",
18
+ "wandb_group": "clean1930s-d12-ctx4096",
19
+ "wandb_tags": "think-dataset-clean-1930s,d12,ratio12,ctx4096,midtrain-schedule,fp8,seed43,variance-replicate",
20
+ "device_type": "",
21
+ "fp8": false,
22
+ "fp8_recipe": "tensorwise",
23
+ "depth": 12,
24
+ "aspect_ratio": 64,
25
+ "head_dim": 128,
26
+ "max_seq_len": 4096,
27
+ "window_pattern": "L",
28
+ "num_iterations": 2520,
29
+ "target_flops": -1.0,
30
+ "target_param_data_ratio": -1.0,
31
+ "device_batch_size": 8,
32
+ "total_batch_size": 524288,
33
+ "embedding_lr": 0.3,
34
+ "unembedding_lr": 0.008,
35
+ "weight_decay": 0.28,
36
+ "matrix_lr": 0.02,
37
+ "scalar_lr": 0.5,
38
+ "warmup_steps": 40,
39
+ "warmdown_ratio": 0.65,
40
+ "final_lr_frac": 0.05,
41
+ "resume_from_step": -1,
42
+ "pretokenized": true,
43
+ "data_dir": "/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/data",
44
+ "tokenizer_dir": "/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/tokenizer",
45
+ "pretokenized_dir": "/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/pretok",
46
+ "mixture_source_dirs": "{\"original\": \"/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/pretok_original\", \"midtrain_r30\": \"/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/pretok_midtrain_r30\", \"midtrain_r60\": \"/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/pretok_midtrain_r60\"}",
47
+ "checkpoint_dir": "/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/base_checkpoints",
48
+ "experiment_id": "1930s-d12-r12-4096ctx-run2",
49
+ "experiment_config": "/content/nanochat_cache/experiments/1930s-d12-r12-4096ctx-run2/config.json",
50
+ "tokenizer_fingerprint": "21e99d5cdeeaa660",
51
+ "git_commit_sha": "c8a82511b832ffd2aa740a0b0453a8a118870493",
52
+ "seed": 43,
53
+ "eval_every": 250,
54
+ "eval_tokens": 2097152,
55
+ "core_metric_every": -1,
56
+ "core_metric_max_per_task": 500,
57
+ "sample_every": -1,
58
+ "save_every": 500,
59
+ "model_tag": "1930s-d12-r12-4096ctx-run2",
60
+ "experiment": {
61
+ "schema_version": 1,
62
+ "stage": "base",
63
+ "experiment_id": "1930s-d12-r12-4096ctx-run2",
64
+ "datasets": {
65
+ "original": {
66
+ "adapter": "parquet_shards",
67
+ "repo": "jbduran/think-dataset-clean-1930s",
68
+ "revision": "main",
69
+ "validation_shard": 472,
70
+ "num_train_shards": 180,
71
+ "download_workers": 4
72
+ },
73
+ "midtrain_r30": {
74
+ "adapter": "parquet_shards",
75
+ "repo": "zachnorton03/think-midtrain",
76
+ "revision": "main",
77
+ "subfolder": "mixed/ratio_30/data",
78
+ "validation_shard": 78,
79
+ "num_train_shards": 40,
80
+ "download_workers": 4
81
+ },
82
+ "midtrain_r60": {
83
+ "adapter": "parquet_shards",
84
+ "repo": "zachnorton03/think-midtrain",
85
+ "revision": "main",
86
+ "subfolder": "mixed/ratio_60/data",
87
+ "validation_shard": 101,
88
+ "num_train_shards": 20,
89
+ "download_workers": 4
90
+ }
91
+ },
92
+ "mixture_schedule": {
93
+ "total_tokens": 1321205760,
94
+ "seed_data": 1234,
95
+ "max_epochs": 4,
96
+ "stages": [
97
+ {
98
+ "name": "base",
99
+ "start_tokens": 0,
100
+ "source": "original"
101
+ },
102
+ {
103
+ "name": "injection",
104
+ "start_tokens": 924844032,
105
+ "source": "midtrain_r30"
106
+ },
107
+ {
108
+ "name": "decay_mix",
109
+ "start_tokens": 1189085184,
110
+ "source": "midtrain_r60"
111
+ }
112
+ ]
113
+ },
114
+ "tokenizer": {
115
+ "mode": "reuse",
116
+ "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
117
+ "vocab_size": 32768
118
+ },
119
+ "pretokenize": {
120
+ "enabled": true,
121
+ "slack": 1.03,
122
+ "require_no_wrap": true,
123
+ "val_tokens": 20971520,
124
+ "shard_tokens": 100000000,
125
+ "tokenizer_threads": 8
126
+ },
127
+ "training": {
128
+ "depth": 12,
129
+ "scaling_params": 110100912,
130
+ "target_param_data_ratio": 12,
131
+ "max_seq_len": 4096,
132
+ "window_pattern": "L",
133
+ "device_batch_size": 8,
134
+ "total_batch_size": 524288,
135
+ "fp8": true,
136
+ "fp8_recipe": "tensorwise",
137
+ "seed": 43,
138
+ "save_every": 500,
139
+ "eval_every": 250,
140
+ "eval_tokens": 2097152,
141
+ "core_metric_every": -1,
142
+ "sample_every": -1
143
+ },
144
+ "artifacts": {
145
+ "repo": "jbduran/think.nano",
146
+ "keep_local_checkpoints": 1
147
+ },
148
+ "wandb": {
149
+ "entity": "jbduran-thinkingmachinesncsu",
150
+ "project": "think.nano",
151
+ "name": "1930s-d12-r12-4096ctx-run2",
152
+ "group": "clean1930s-d12-ctx4096",
153
+ "tags": [
154
+ "think-dataset-clean-1930s",
155
+ "d12",
156
+ "ratio12",
157
+ "ctx4096",
158
+ "midtrain-schedule",
159
+ "fp8",
160
+ "seed43",
161
+ "variance-replicate"
162
+ ]
163
+ },
164
+ "config_fingerprint": "8d2026ebce2c42d6",
165
+ "artifact_path": "experiments/1930s-d12-r12-4096ctx-run2"
166
+ },
167
+ "stage": "base",
168
+ "base_experiment_id": "1930s-d12-r12-4096ctx-run2",
169
+ "parent_experiment_id": null,
170
+ "parent_checkpoint_step": null,
171
+ "config_fingerprint": "8d2026ebce2c42d6",
172
+ "fp8_unavailable_reason": "NVIDIA A100-SXM4-40GB is SM 80; FP8 requires SM 89+ (Ada/Hopper)"
173
+ },
174
+ "device_batch_size": 8,
175
+ "max_seq_len": 4096,
176
+ "total_batch_size": 524288,
177
+ "dataloader_state_dict": {
178
+ "file_idx": 1,
179
+ "pos": 32157377,
180
+ "epoch": 1,
181
+ "pq_idx": 1,
182
+ "rg_idx": 32157377,
183
+ "mixture": {
184
+ "cursors": {
185
+ "original": {
186
+ "file_idx": 9,
187
+ "pos": 24872256,
188
+ "epoch": 1,
189
+ "pq_idx": 9,
190
+ "rg_idx": 24872256
191
+ },
192
+ "midtrain_r30": {
193
+ "file_idx": 2,
194
+ "pos": 64249216,
195
+ "epoch": 1,
196
+ "pq_idx": 2,
197
+ "rg_idx": 64249216
198
+ },
199
+ "midtrain_r60": {
200
+ "file_idx": 1,
201
+ "pos": 32157377,
202
+ "epoch": 1,
203
+ "pq_idx": 1,
204
+ "rg_idx": 32157377
205
+ }
206
+ },
207
+ "cumulative_tokens": 1321238528,
208
+ "source_tokens": {
209
+ "original": 924844032,
210
+ "midtrain_r30": 264241152,
211
+ "midtrain_r60": 132153344
212
+ },
213
+ "active_stage_idx": 2
214
+ }
215
+ },
216
+ "loop_state": {
217
+ "min_val_bpb": 1.0879491465161388,
218
+ "smooth_train_loss": 3.0810559880062787,
219
+ "total_training_time": 7626.332884550095,
220
+ "stage_training_flops": 1471281923924951040,
221
+ "inherited_parent_flops": 0.0,
222
+ "cumulative_pipeline_training_flops": 1471281923924951040
223
+ }
224
+ }