Hugging Face's logo

jbduran
/
bartholomew

PyTorch
English
nanochat
language-model
pre-1930
historical
vintage
Model card Files
xet
 Community
3
jbduran commited on
Commit
03225f6
·
verified ·
1 Parent(s): 9ac6718

Slim bart to the final model; experiments moved to jbduran/bart-experiments

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_000500.json +0 -61
  2. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_001000.json +0 -61
  3. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_001500.json +0 -61
  4. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_002000.json +0 -61
  5. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_002500.json +0 -61
  6. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_003000.json +0 -61
  7. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_003500.json +0 -61
  8. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_004000.json +0 -61
  9. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_004200.json +0 -61
  10. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_000500.pt +0 -3
  11. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_001000.pt +0 -3
  12. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_001500.pt +0 -3
  13. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_002000.pt +0 -3
  14. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_002500.pt +0 -3
  15. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_003000.pt +0 -3
  16. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_003500.pt +0 -3
  17. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_004000.pt +0 -3
  18. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_004200.pt +0 -3
  19. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_000500_rank0.pt +0 -3
  20. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_001000_rank0.pt +0 -3
  21. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_001500_rank0.pt +0 -3
  22. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_002000_rank0.pt +0 -3
  23. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_002500_rank0.pt +0 -3
  24. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_003000_rank0.pt +0 -3
  25. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_003500_rank0.pt +0 -3
  26. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_004000_rank0.pt +0 -3
  27. archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_004200_rank0.pt +0 -3
  28. archive/pre-lineage-v1/base_checkpoints/d12/meta_000500.json +0 -61
  29. archive/pre-lineage-v1/base_checkpoints/d12/meta_001000.json +0 -61
  30. archive/pre-lineage-v1/base_checkpoints/d12/meta_001500.json +0 -61
  31. archive/pre-lineage-v1/base_checkpoints/d12/meta_002000.json +0 -61
  32. archive/pre-lineage-v1/base_checkpoints/d12/meta_002362.json +0 -61
  33. archive/pre-lineage-v1/base_checkpoints/d12/model_000500.pt +0 -3
  34. archive/pre-lineage-v1/base_checkpoints/d12/model_001000.pt +0 -3
  35. archive/pre-lineage-v1/base_checkpoints/d12/model_001500.pt +0 -3
  36. archive/pre-lineage-v1/base_checkpoints/d12/model_002000.pt +0 -3
  37. archive/pre-lineage-v1/base_checkpoints/d12/model_002362.pt +0 -3
  38. archive/pre-lineage-v1/base_checkpoints/d12/optim_000500_rank0.pt +0 -3
  39. archive/pre-lineage-v1/base_checkpoints/d12/optim_001000_rank0.pt +0 -3
  40. archive/pre-lineage-v1/base_checkpoints/d12/optim_001500_rank0.pt +0 -3
  41. archive/pre-lineage-v1/base_checkpoints/d12/optim_002000_rank0.pt +0 -3
  42. archive/pre-lineage-v1/base_checkpoints/d12/optim_002362_rank0.pt +0 -3
  43. archive/pre-lineage-v1/chatsft_checkpoints/d12/meta_001065.json +0 -38
  44. archive/pre-lineage-v1/chatsft_checkpoints/d12/model_001065.pt +0 -3
  45. archive/pre-lineage-v1/chatsft_checkpoints/d12/optim_001065_rank0.pt +0 -3
  46. archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/meta_000500.json +0 -125
  47. archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/meta_001000.json +0 -125
  48. archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/meta_001500.json +0 -125
  49. archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/model_000500.pt +0 -3
  50. archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/model_001000.pt +0 -3
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_000500.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 500,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 2,
51
- "pos": 62184769,
52
- "epoch": 1,
53
- "pq_idx": 2,
54
- "rg_idx": 62184769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.5891939434241213,
59
- "total_training_time": 1312.64857006073
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_001000.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 1000,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 5,
51
- "pos": 24336769,
52
- "epoch": 1,
53
- "pq_idx": 5,
54
- "rg_idx": 24336769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.4868806829553485,
59
- "total_training_time": 2654.1426842212677
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_001500.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 1500,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 7,
51
- "pos": 86488769,
52
- "epoch": 1,
53
- "pq_idx": 7,
54
- "rg_idx": 86488769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.410999880107203,
59
- "total_training_time": 3995.465323448181
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_002000.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 2000,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 10,
51
- "pos": 48640769,
52
- "epoch": 1,
53
- "pq_idx": 10,
54
- "rg_idx": 48640769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.452807694528099,
59
- "total_training_time": 5336.5537366867065
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_002500.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 2500,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 13,
51
- "pos": 10792769,
52
- "epoch": 1,
53
- "pq_idx": 13,
54
- "rg_idx": 10792769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.327396609246394,
59
- "total_training_time": 6677.6987290382385
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_003000.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 3000,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 15,
51
- "pos": 72944769,
52
- "epoch": 1,
53
- "pq_idx": 15,
54
- "rg_idx": 72944769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.048554485031434,
59
- "total_training_time": 8018.697687149048
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_003500.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 3500,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 18,
51
- "pos": 35096769,
52
- "epoch": 1,
53
- "pq_idx": 18,
54
- "rg_idx": 35096769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.0863692227626416,
59
- "total_training_time": 9359.67183303833
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_004000.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 4000,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 20,
51
- "pos": 97248769,
52
- "epoch": 1,
53
- "pq_idx": 20,
54
- "rg_idx": 97248769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 2.7993042062101186,
59
- "total_training_time": 10700.389838218689
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/meta_004200.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 4200,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 20.0,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": "d12-ratio20"
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 22,
51
- "pos": 2109569,
52
- "epoch": 1,
53
- "pq_idx": 22,
54
- "rg_idx": 2109569
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 2.8461822660242255,
59
- "total_training_time": 11236.57096004486
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_000500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:00e14e7ad4637b08c6e5f649ebc7de7ed0d9fcb39a293b7b088a13c2ebb549f8
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_001000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:01e2a9466841de94c8c3a654a257c842c1764d3ac4cd79cf478f9f2b24859e7c
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_001500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:03fb5d4241142f86b69396fe200295929df1b4bcc2ba5d4cc464c9627cf0a530
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_002000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c4db8d9fa2b5fa4ca14f65dfd8d5356c691c5c8772e839555f0fec6f8d300269
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_002500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:abb54590a71621be44e418b133a42088aed1d84872a9726e9aa4ac8b7e418788
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_003000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0a96aa9105d455f741602d43d807dd0b2732823c9c8be3465717f7090aa469eb
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_003500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:bab4a0741306ce5945c9a86dabd233081b836f0e1f814dd1d9363cc1b9cce950
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_004000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6c967e927215ebca6af3f49011d98fd8fa4487c55e78485271878e382b7293d5
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/model_004200.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1dfab1c19f9bbd3dd27ed859b70cfd242d5ebaef734985dbef78d4998493a6f6
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_000500_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:62b942e737b779c2dbb3b14246ffc951fb9dce816e10e7ada936b81093faaca7
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_001000_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f5fe23783669dcb79dda4bc0a66e2c536d9d3055fbcaf07840662e2c5b8f6950
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_001500_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9dcce76cabc6cdd8083af5f064ae391d8b84fc96d2608ab6740fffc2b1e2f5f2
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_002000_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4dc97c55d9aa7e22cd38dcfc298b42066aeda741daf7d1b508575ac62c0e32de
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_002500_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:2550badb9d208d9f2e51765bafd6789cc2f9ee8b6fd5880e25790d427deb9556
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_003000_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8ea5ec19b0375980f64cbdbb9709d040a01e2f546ebdcc380246f8c69d26b242
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_003500_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9465372371e5d248972fe855b83234cbf5fde5bcf208eb7e2faf9f51cf207def
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_004000_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c27aa4c1fea51a61cd5aef3654780852040f59b347b9f18c7a9f29620664b677
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12-ratio20/optim_004200_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3a71d1f755bf5e13a1bd7741a3a1bbc1d9807468b5190c4e2380b91e0116eee0
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/meta_000500.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 500,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 11.25,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": null
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 2,
51
- "pos": 62184769,
52
- "epoch": 1,
53
- "pq_idx": 2,
54
- "rg_idx": 62184769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.5893779623775406,
59
- "total_training_time": 1290.4532148838043
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/meta_001000.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 1000,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 11.25,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": null
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 5,
51
- "pos": 24336769,
52
- "epoch": 1,
53
- "pq_idx": 5,
54
- "rg_idx": 24336769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.4419165825253133,
59
- "total_training_time": 2609.577807664871
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/meta_001500.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 1500,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 11.25,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": null
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 7,
51
- "pos": 86488769,
52
- "epoch": 1,
53
- "pq_idx": 7,
54
- "rg_idx": 86488769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.2465426140603015,
59
- "total_training_time": 3929.598204135895
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/meta_002000.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 2000,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 11.25,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": null
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 10,
51
- "pos": 48640769,
52
- "epoch": 1,
53
- "pq_idx": 10,
54
- "rg_idx": 48640769
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.2442641345309373,
59
- "total_training_time": 5249.494728565216
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/meta_002362.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "step": 2362,
3
- "val_bpb": null,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "fp8": false,
17
- "fp8_recipe": "tensorwise",
18
- "depth": 12,
19
- "aspect_ratio": 64,
20
- "head_dim": 128,
21
- "max_seq_len": 2048,
22
- "window_pattern": "L",
23
- "num_iterations": -1,
24
- "target_flops": -1.0,
25
- "target_param_data_ratio": 11.25,
26
- "device_batch_size": 16,
27
- "total_batch_size": -1,
28
- "embedding_lr": 0.3,
29
- "unembedding_lr": 0.008,
30
- "weight_decay": 0.28,
31
- "matrix_lr": 0.02,
32
- "scalar_lr": 0.5,
33
- "warmup_steps": 40,
34
- "warmdown_ratio": 0.65,
35
- "final_lr_frac": 0.05,
36
- "resume_from_step": -1,
37
- "pretokenized": true,
38
- "eval_every": -1,
39
- "eval_tokens": 41943040,
40
- "core_metric_every": -1,
41
- "core_metric_max_per_task": 500,
42
- "sample_every": -1,
43
- "save_every": 500,
44
- "model_tag": null
45
- },
46
- "device_batch_size": 16,
47
- "max_seq_len": 2048,
48
- "total_batch_size": 524288,
49
- "dataloader_state_dict": {
50
- "file_idx": 12,
51
- "pos": 38438817,
52
- "epoch": 1,
53
- "pq_idx": 12,
54
- "rg_idx": 38438817
55
- },
56
- "loop_state": {
57
- "min_val_bpb": Infinity,
58
- "smooth_train_loss": 3.074421420856799,
59
- "total_training_time": 6205.646646976471
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/model_000500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:21fbdff366e5db3fa2f254b4b98c763cc70ba95722242fa032d8d21b956b694f
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/model_001000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6f0a862044e51b8cac250cf1e4f26af7f8347fd887b8c30af08addb879b6494d
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/model_001500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:326bddc26c100de33310f9164dc873af6099a9b7760527a8707c256988a8ec7a
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/model_002000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a5d101975722cffc43eb4f35aa967a5afd397b159da3521e5a1acbd3819d466e
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/model_002362.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:04331c52ed8fa7259f350e4ec72d0dd6602451cfd75a5773a4c17ac5c141ea7e
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/optim_000500_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d39a131089a476a202ae932f21d9398b225d0b8e4147a58fbdff797914d34976
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/optim_001000_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e4971566eb3d4aa5d5fe29b3a3b77f56581b61713e5c1d162debb8a409c02118
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/optim_001500_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:05ce87d44bfd6e9b4d4c1e643a2a5aa4b169119913ccbdf3625d7cfa7313b342
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/optim_002000_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1b58f68fd887360ce99e8456aecf706b1bcf5c8210194721389698ec658237b1
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/base_checkpoints/d12/optim_002362_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:09aa5632a4b33981a1b2fbd97254d0050ecb7916d0c242f1d195c0726be314d7
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/chatsft_checkpoints/d12/meta_001065.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "step": 1065,
3
- "val_bpb": 0.39271438585109175,
4
- "model_config": {
5
- "sequence_len": 2048,
6
- "vocab_size": 32768,
7
- "n_layer": 12,
8
- "n_head": 6,
9
- "n_kv_head": 6,
10
- "n_embd": 768,
11
- "window_pattern": "L"
12
- },
13
- "user_config": {
14
- "run": "dummy",
15
- "device_type": "",
16
- "model_tag": "d12",
17
- "model_step": null,
18
- "load_optimizer": 1,
19
- "num_iterations": -1,
20
- "max_seq_len": null,
21
- "device_batch_size": 8,
22
- "total_batch_size": null,
23
- "embedding_lr": null,
24
- "unembedding_lr": null,
25
- "matrix_lr": null,
26
- "init_lr_frac": 0.8,
27
- "warmup_ratio": 0.0,
28
- "warmdown_ratio": 0.5,
29
- "final_lr_frac": 0.0,
30
- "eval_every": -1,
31
- "eval_tokens": 20971520,
32
- "chatcore_every": -1,
33
- "chatcore_max_cat": -1,
34
- "chatcore_max_sample": 24,
35
- "mmlu_epochs": 3,
36
- "gsm8k_epochs": 4
37
- }
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/chatsft_checkpoints/d12/model_001065.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f1ef8886ea2cbd820baa9bc98368f99673efb1f5fdbd082196d573b291111bda
3
- size 792761399
 
 
 
 
archive/pre-lineage-v1/chatsft_checkpoints/d12/optim_001065_rank0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:db8c9df6e288618ed6362102e29edc3731267b4ff382154a55709b4d5a14f1d4
3
- size 1246165237
 
 
 
 
archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/meta_000500.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "step": 500,
3
- "experiment_id": "climbmix-d12-r12-baseline-v2",
4
- "val_bpb": 1.039594309585361,
5
- "model_config": {
6
- "sequence_len": 2048,
7
- "vocab_size": 32768,
8
- "n_layer": 12,
9
- "n_head": 6,
10
- "n_kv_head": 6,
11
- "n_embd": 768,
12
- "window_pattern": "L"
13
- },
14
- "user_config": {
15
- "run": "climbmix-d12-r12-baseline-v2",
16
- "wandb_run_id": "412c294c",
17
- "wandb_group": "climbmix-d12",
18
- "wandb_tags": "climbmix,d12,baseline,ratio12",
19
- "device_type": "",
20
- "fp8": false,
21
- "fp8_recipe": "tensorwise",
22
- "depth": 12,
23
- "aspect_ratio": 64,
24
- "head_dim": 128,
25
- "max_seq_len": 2048,
26
- "window_pattern": "L",
27
- "num_iterations": -1,
28
- "target_flops": -1.0,
29
- "target_param_data_ratio": 12.0,
30
- "device_batch_size": 16,
31
- "total_batch_size": 524288,
32
- "embedding_lr": 0.3,
33
- "unembedding_lr": 0.008,
34
- "weight_decay": 0.28,
35
- "matrix_lr": 0.02,
36
- "scalar_lr": 0.5,
37
- "warmup_steps": 40,
38
- "warmdown_ratio": 0.65,
39
- "final_lr_frac": 0.05,
40
- "resume_from_step": -1,
41
- "pretokenized": true,
42
- "data_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/data",
43
- "tokenizer_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/tokenizer",
44
- "pretokenized_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/pretok",
45
- "checkpoint_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints",
46
- "experiment_id": "climbmix-d12-r12-baseline-v2",
47
- "experiment_config": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/config.json",
48
- "eval_every": 250,
49
- "eval_tokens": 2097152,
50
- "core_metric_every": -1,
51
- "core_metric_max_per_task": 500,
52
- "sample_every": -1,
53
- "save_every": 500,
54
- "model_tag": "climbmix-d12-r12-baseline-v2",
55
- "experiment": {
56
- "experiment_id": "climbmix-d12-r12-baseline-v2",
57
- "dataset": {
58
- "adapter": "parquet_shards",
59
- "repo": "karpathy/climbmix-400b-shuffle",
60
- "revision": "main",
61
- "base_url": "https://huggingface.co/datasets/karpathy/climbmix-400b-shuffle/resolve/main",
62
- "validation_shard": 6542,
63
- "num_train_shards": 8,
64
- "download_workers": 4
65
- },
66
- "tokenizer": {
67
- "mode": "train",
68
- "max_chars": 2000000000,
69
- "doc_cap": 10000,
70
- "vocab_size": 32768
71
- },
72
- "pretokenize": {
73
- "enabled": true,
74
- "slack": 1.03,
75
- "val_tokens": 20971520,
76
- "shard_tokens": 100000000,
77
- "tokenizer_threads": 8
78
- },
79
- "training": {
80
- "depth": 12,
81
- "scaling_params": 110100912,
82
- "target_param_data_ratio": 12.0,
83
- "window_pattern": "L",
84
- "device_batch_size": 16,
85
- "total_batch_size": 524288,
86
- "save_every": 500,
87
- "eval_every": 250,
88
- "eval_tokens": 2097152,
89
- "core_metric_every": -1,
90
- "sample_every": -1
91
- },
92
- "storage": {
93
- "hf_model_repo": "jbduran/think-nanochat-d12",
94
- "path": "experiments/climbmix-d12-r12-baseline-v2"
95
- },
96
- "wandb": {
97
- "entity": "jbduran-thinkingmachinesncsu",
98
- "project": "think.nano",
99
- "name": "climbmix-d12-r12-baseline-v2",
100
- "group": "climbmix-d12",
101
- "tags": [
102
- "climbmix",
103
- "d12",
104
- "baseline",
105
- "ratio12"
106
- ]
107
- }
108
- }
109
- },
110
- "device_batch_size": 16,
111
- "max_seq_len": 2048,
112
- "total_batch_size": 524288,
113
- "dataloader_state_dict": {
114
- "file_idx": 2,
115
- "pos": 62184769,
116
- "epoch": 1,
117
- "pq_idx": 2,
118
- "rg_idx": 62184769
119
- },
120
- "loop_state": {
121
- "min_val_bpb": 1.039594309585361,
122
- "smooth_train_loss": 3.4405327881291092,
123
- "total_training_time": 1309.0715219974518
124
- }
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/meta_001000.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "step": 1000,
3
- "experiment_id": "climbmix-d12-r12-baseline-v2",
4
- "val_bpb": 0.9812561487708248,
5
- "model_config": {
6
- "sequence_len": 2048,
7
- "vocab_size": 32768,
8
- "n_layer": 12,
9
- "n_head": 6,
10
- "n_kv_head": 6,
11
- "n_embd": 768,
12
- "window_pattern": "L"
13
- },
14
- "user_config": {
15
- "run": "climbmix-d12-r12-baseline-v2",
16
- "wandb_run_id": "412c294c",
17
- "wandb_group": "climbmix-d12",
18
- "wandb_tags": "climbmix,d12,baseline,ratio12",
19
- "device_type": "",
20
- "fp8": false,
21
- "fp8_recipe": "tensorwise",
22
- "depth": 12,
23
- "aspect_ratio": 64,
24
- "head_dim": 128,
25
- "max_seq_len": 2048,
26
- "window_pattern": "L",
27
- "num_iterations": -1,
28
- "target_flops": -1.0,
29
- "target_param_data_ratio": 12.0,
30
- "device_batch_size": 16,
31
- "total_batch_size": 524288,
32
- "embedding_lr": 0.3,
33
- "unembedding_lr": 0.008,
34
- "weight_decay": 0.28,
35
- "matrix_lr": 0.02,
36
- "scalar_lr": 0.5,
37
- "warmup_steps": 40,
38
- "warmdown_ratio": 0.65,
39
- "final_lr_frac": 0.05,
40
- "resume_from_step": -1,
41
- "pretokenized": true,
42
- "data_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/data",
43
- "tokenizer_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/tokenizer",
44
- "pretokenized_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/pretok",
45
- "checkpoint_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints",
46
- "experiment_id": "climbmix-d12-r12-baseline-v2",
47
- "experiment_config": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/config.json",
48
- "eval_every": 250,
49
- "eval_tokens": 2097152,
50
- "core_metric_every": -1,
51
- "core_metric_max_per_task": 500,
52
- "sample_every": -1,
53
- "save_every": 500,
54
- "model_tag": "climbmix-d12-r12-baseline-v2",
55
- "experiment": {
56
- "experiment_id": "climbmix-d12-r12-baseline-v2",
57
- "dataset": {
58
- "adapter": "parquet_shards",
59
- "repo": "karpathy/climbmix-400b-shuffle",
60
- "revision": "main",
61
- "base_url": "https://huggingface.co/datasets/karpathy/climbmix-400b-shuffle/resolve/main",
62
- "validation_shard": 6542,
63
- "num_train_shards": 8,
64
- "download_workers": 4
65
- },
66
- "tokenizer": {
67
- "mode": "train",
68
- "max_chars": 2000000000,
69
- "doc_cap": 10000,
70
- "vocab_size": 32768
71
- },
72
- "pretokenize": {
73
- "enabled": true,
74
- "slack": 1.03,
75
- "val_tokens": 20971520,
76
- "shard_tokens": 100000000,
77
- "tokenizer_threads": 8
78
- },
79
- "training": {
80
- "depth": 12,
81
- "scaling_params": 110100912,
82
- "target_param_data_ratio": 12.0,
83
- "window_pattern": "L",
84
- "device_batch_size": 16,
85
- "total_batch_size": 524288,
86
- "save_every": 500,
87
- "eval_every": 250,
88
- "eval_tokens": 2097152,
89
- "core_metric_every": -1,
90
- "sample_every": -1
91
- },
92
- "storage": {
93
- "hf_model_repo": "jbduran/think-nanochat-d12",
94
- "path": "experiments/climbmix-d12-r12-baseline-v2"
95
- },
96
- "wandb": {
97
- "entity": "jbduran-thinkingmachinesncsu",
98
- "project": "think.nano",
99
- "name": "climbmix-d12-r12-baseline-v2",
100
- "group": "climbmix-d12",
101
- "tags": [
102
- "climbmix",
103
- "d12",
104
- "baseline",
105
- "ratio12"
106
- ]
107
- }
108
- }
109
- },
110
- "device_batch_size": 16,
111
- "max_seq_len": 2048,
112
- "total_batch_size": 524288,
113
- "dataloader_state_dict": {
114
- "file_idx": 0,
115
- "pos": 93414403,
116
- "epoch": 2,
117
- "pq_idx": 0,
118
- "rg_idx": 93414403
119
- },
120
- "loop_state": {
121
- "min_val_bpb": 0.9812561487708248,
122
- "smooth_train_loss": 3.1852821933493254,
123
- "total_training_time": 2645.4519686698914
124
- }
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/meta_001500.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "step": 1500,
3
- "experiment_id": "climbmix-d12-r12-baseline-v2",
4
- "val_bpb": 0.9399637061635419,
5
- "model_config": {
6
- "sequence_len": 2048,
7
- "vocab_size": 32768,
8
- "n_layer": 12,
9
- "n_head": 6,
10
- "n_kv_head": 6,
11
- "n_embd": 768,
12
- "window_pattern": "L"
13
- },
14
- "user_config": {
15
- "run": "climbmix-d12-r12-baseline-v2",
16
- "wandb_run_id": "412c294c",
17
- "wandb_group": "climbmix-d12",
18
- "wandb_tags": "climbmix,d12,baseline,ratio12",
19
- "device_type": "",
20
- "fp8": false,
21
- "fp8_recipe": "tensorwise",
22
- "depth": 12,
23
- "aspect_ratio": 64,
24
- "head_dim": 128,
25
- "max_seq_len": 2048,
26
- "window_pattern": "L",
27
- "num_iterations": -1,
28
- "target_flops": -1.0,
29
- "target_param_data_ratio": 12.0,
30
- "device_batch_size": 16,
31
- "total_batch_size": 524288,
32
- "embedding_lr": 0.3,
33
- "unembedding_lr": 0.008,
34
- "weight_decay": 0.28,
35
- "matrix_lr": 0.02,
36
- "scalar_lr": 0.5,
37
- "warmup_steps": 40,
38
- "warmdown_ratio": 0.65,
39
- "final_lr_frac": 0.05,
40
- "resume_from_step": -1,
41
- "pretokenized": true,
42
- "data_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/data",
43
- "tokenizer_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/tokenizer",
44
- "pretokenized_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/pretok",
45
- "checkpoint_dir": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints",
46
- "experiment_id": "climbmix-d12-r12-baseline-v2",
47
- "experiment_config": "/content/nanochat_cache/experiments/climbmix-d12-r12-baseline-v2/config.json",
48
- "eval_every": 250,
49
- "eval_tokens": 2097152,
50
- "core_metric_every": -1,
51
- "core_metric_max_per_task": 500,
52
- "sample_every": -1,
53
- "save_every": 500,
54
- "model_tag": "climbmix-d12-r12-baseline-v2",
55
- "experiment": {
56
- "experiment_id": "climbmix-d12-r12-baseline-v2",
57
- "dataset": {
58
- "adapter": "parquet_shards",
59
- "repo": "karpathy/climbmix-400b-shuffle",
60
- "revision": "main",
61
- "base_url": "https://huggingface.co/datasets/karpathy/climbmix-400b-shuffle/resolve/main",
62
- "validation_shard": 6542,
63
- "num_train_shards": 8,
64
- "download_workers": 4
65
- },
66
- "tokenizer": {
67
- "mode": "train",
68
- "max_chars": 2000000000,
69
- "doc_cap": 10000,
70
- "vocab_size": 32768
71
- },
72
- "pretokenize": {
73
- "enabled": true,
74
- "slack": 1.03,
75
- "val_tokens": 20971520,
76
- "shard_tokens": 100000000,
77
- "tokenizer_threads": 8
78
- },
79
- "training": {
80
- "depth": 12,
81
- "scaling_params": 110100912,
82
- "target_param_data_ratio": 12.0,
83
- "window_pattern": "L",
84
- "device_batch_size": 16,
85
- "total_batch_size": 524288,
86
- "save_every": 500,
87
- "eval_every": 250,
88
- "eval_tokens": 2097152,
89
- "core_metric_every": -1,
90
- "sample_every": -1
91
- },
92
- "storage": {
93
- "hf_model_repo": "jbduran/think-nanochat-d12",
94
- "path": "experiments/climbmix-d12-r12-baseline-v2"
95
- },
96
- "wandb": {
97
- "entity": "jbduran-thinkingmachinesncsu",
98
- "project": "think.nano",
99
- "name": "climbmix-d12-r12-baseline-v2",
100
- "group": "climbmix-d12",
101
- "tags": [
102
- "climbmix",
103
- "d12",
104
- "baseline",
105
- "ratio12"
106
- ]
107
- }
108
- }
109
- },
110
- "device_batch_size": 16,
111
- "max_seq_len": 2048,
112
- "total_batch_size": 524288,
113
- "dataloader_state_dict": {
114
- "file_idx": 3,
115
- "pos": 55566403,
116
- "epoch": 2,
117
- "pq_idx": 3,
118
- "rg_idx": 55566403
119
- },
120
- "loop_state": {
121
- "min_val_bpb": 0.9399637061635419,
122
- "smooth_train_loss": 3.0342439616648695,
123
- "total_training_time": 3982.5033581256866
124
- }
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/model_000500.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:85c521a72ea4400bdbe8b2eb173deae14f7957a9075268bbbd1a0d4690223b60
3
- size 792761690
 
 
 
 
archive/pre-lineage-v1/experiments/climbmix-d12-r12-baseline-v2/base_checkpoints/model_001000.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e3c2f15fb478ba9da622f478a05f978c8f156c9c19a192c957051b188a8c83b0
3
- size 792761690