Hugging Face's logo

jbduran
/
bartholomew-sft

PyTorch
English
nanochat
language-model
pre-1930
historical
vintage
sft
conversational
Model card Files
xet
 Community
jbduran commited on
Commit
2c31e78
·
verified ·
1 Parent(s): a1159aa

Import SFT model: pre1930-curriculum-c3-robust-v2

Browse files
Files changed (7) hide show
  1. checkpoints/eval_metrics.json +31 -0
  2. checkpoints/meta_000042.json +178 -0
  3. checkpoints/model_000042.pt +3 -0
  4. checkpoints/optim_000042_rank0.pt +3 -0
  5. config.json +95 -0
  6. evals/chatcore.json +29 -0
  7. run.json +11 -0
checkpoints/eval_metrics.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
3
+ "recipe": "curriculum",
4
+ "step": 42,
5
+ "val_bpb": 0.7037352212342954,
6
+ "min_val_bpb": 0.7037352212342954,
7
+ "per_route_bpb": {},
8
+ "per_domain_bpb": {},
9
+ "curriculum_summary": null,
10
+ "chatcore": {
11
+ "chatcore_metric": -0.011321608241655831,
12
+ "chatcore_cat": -0.018869347069426386,
13
+ "suite": {
14
+ "name": "karpathy",
15
+ "tasks": [
16
+ "ARC-Easy",
17
+ "ARC-Challenge",
18
+ "MMLU",
19
+ "GSM8K",
20
+ "HumanEval"
21
+ ],
22
+ "max_generative_problems": null,
23
+ "generative_answer_format": null
24
+ },
25
+ "ARC-Easy": 0.24579124579124578,
26
+ "ARC-Challenge": 0.23037542662116042,
27
+ "MMLU": 0.23137729668138443,
28
+ "GSM8K": 0.0,
29
+ "HumanEval": 0.0
30
+ }
31
+ }
checkpoints/meta_000042.json ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 42,
3
+ "training_complete": true,
4
+ "val_bpb": 0.7037352212342954,
5
+ "model_config": {
6
+ "sequence_len": 4096,
7
+ "vocab_size": 32768,
8
+ "n_layer": 32,
9
+ "n_head": 16,
10
+ "n_kv_head": 16,
11
+ "n_embd": 2048,
12
+ "window_pattern": "SSSL"
13
+ },
14
+ "user_config": {
15
+ "run": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
16
+ "wandb_run_id": "036b1c71",
17
+ "wandb_group": "think-d32",
18
+ "wandb_tags": "sft,curriculum,c3,scale-max,staged,robustness,noise,d32",
19
+ "device_type": "",
20
+ "model_tag": null,
21
+ "model_step": null,
22
+ "base_checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/base_checkpoints",
23
+ "base_step": 9600,
24
+ "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints",
25
+ "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer",
26
+ "resume_from_step": null,
27
+ "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
28
+ "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json",
29
+ "parent_cumulative_flops": 2.3103503196928082e+20,
30
+ "tokenizer_fingerprint": "21e99d5cdeeaa660",
31
+ "git_commit_sha": "04bb043ad61d4db3e9022c422302df7b8f2dc0c9",
32
+ "load_optimizer": 0,
33
+ "num_iterations": -1,
34
+ "max_seq_len": null,
35
+ "device_batch_size": 2,
36
+ "total_batch_size": null,
37
+ "embedding_lr": null,
38
+ "unembedding_lr": null,
39
+ "matrix_lr": null,
40
+ "init_lr_frac": 0.8,
41
+ "warmup_ratio": 0.03,
42
+ "warmdown_ratio": 0.5,
43
+ "final_lr_frac": 0.0,
44
+ "eval_every": 400,
45
+ "eval_tokens": 20971520,
46
+ "chatcore_every": 100000,
47
+ "chatcore_max_cat": -1,
48
+ "chatcore_max_sample": 32,
49
+ "save_every": -1,
50
+ "recipe": "curriculum",
51
+ "curriculum_config": "",
52
+ "pre1930_epochs": 5,
53
+ "mmlu_epochs": 3,
54
+ "gsm8k_epochs": 4,
55
+ "authentic_epochs": 0,
56
+ "knowledge_qa_epochs": 0,
57
+ "multiturn_qa_epochs": 0,
58
+ "reasoning_qa_epochs": 0,
59
+ "stem_reasoning_epochs": 0,
60
+ "narrative_grounded_epochs": 0,
61
+ "narrative_fiction_epochs": 0,
62
+ "opinion_qa_epochs": 0,
63
+ "how_to_qa_epochs": 0,
64
+ "verse_qa_epochs": 0,
65
+ "composition_qa_epochs": 0,
66
+ "resolved_experiment_config": {
67
+ "schema_version": 1,
68
+ "stage": "sft",
69
+ "experiment_suffix": "pre1930-curriculum-c3-robust-v2",
70
+ "data": {
71
+ "recipe": "curriculum",
72
+ "curriculum": {
73
+ "name": "C3Rv2",
74
+ "mode": "staged",
75
+ "threshold_default": 80,
76
+ "stages": [
77
+ {
78
+ "routes": [
79
+ "knowledge_qa"
80
+ ],
81
+ "authentic": "single"
82
+ },
83
+ {
84
+ "routes": [
85
+ "reasoning_qa",
86
+ "stem_reasoning",
87
+ "how_to_qa",
88
+ "opinion_qa",
89
+ "composition_qa",
90
+ "verse_qa"
91
+ ],
92
+ "calibration_qa": true
93
+ },
94
+ {
95
+ "routes": [
96
+ "multiturn_qa",
97
+ "narrative_grounded",
98
+ "narrative_fiction"
99
+ ],
100
+ "authentic": "multi"
101
+ }
102
+ ],
103
+ "noise": {
104
+ "rate": 0.3
105
+ },
106
+ "robustness": {
107
+ "stage": 0,
108
+ "epochs": 1,
109
+ "routes": {
110
+ "conversation_qa": {
111
+ "count": null
112
+ },
113
+ "unparseable_qa": {
114
+ "count": null
115
+ },
116
+ "typo_qa": {
117
+ "count": null
118
+ },
119
+ "era_qa": {
120
+ "count": null
121
+ },
122
+ "conversation_multiturn": {
123
+ "count": null
124
+ }
125
+ }
126
+ }
127
+ }
128
+ },
129
+ "training": {
130
+ "num_iterations": -1,
131
+ "load_optimizer": 0,
132
+ "device_batch_size": 2,
133
+ "warmup_ratio": 0.03,
134
+ "eval_every": 400,
135
+ "chatcore_every": 100000,
136
+ "chatcore_max_sample": 32,
137
+ "save_every": -1
138
+ },
139
+ "artifacts": {
140
+ "repo": "jbduran/think.nano"
141
+ },
142
+ "wandb": {
143
+ "enabled": true,
144
+ "entity": "jbduran-thinkingmachinesncsu",
145
+ "project": "think.nano",
146
+ "group": "think-d32",
147
+ "tags": [
148
+ "sft",
149
+ "curriculum",
150
+ "c3",
151
+ "scale-max",
152
+ "staged",
153
+ "robustness",
154
+ "noise",
155
+ "d32"
156
+ ]
157
+ },
158
+ "config_fingerprint": "4462a389dc98cd5d",
159
+ "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2"
160
+ },
161
+ "stage": "sft",
162
+ "base_experiment_id": null,
163
+ "parent_experiment_id": null,
164
+ "parent_checkpoint_step": null,
165
+ "config_fingerprint": "4462a389dc98cd5d"
166
+ },
167
+ "loop_state": {
168
+ "step": 42,
169
+ "total_training_time": 1503.5729036331177,
170
+ "min_val_bpb": 0.7037352212342954,
171
+ "smooth_train_loss": 1.8593226867746961,
172
+ "mfu": 51.82901425419851,
173
+ "tok_per_sec": 44667,
174
+ "stage_training_flops": 1.0107782648656036e+18,
175
+ "inherited_parent_flops": 2.3103503196928082e+20,
176
+ "cumulative_pipeline_training_flops": 2.3204581023414642e+20
177
+ }
178
+ }
checkpoints/model_000042.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6a9bc83e3853f0a7ad8b27cc45388d727cc5976837f514d61ea59f80d7f0348
3
+ size 8992694653
checkpoints/optim_000042_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34c0871ec1aed5ced55a463ea1b827e4ef80e286735cc42512ced11f8c579fe4
3
+ size 11545903753
config.json ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "stage": "sft",
4
+ "experiment_suffix": "pre1930-curriculum-c3-robust-v2",
5
+ "data": {
6
+ "recipe": "curriculum",
7
+ "curriculum": {
8
+ "name": "C3Rv2",
9
+ "mode": "staged",
10
+ "threshold_default": 80,
11
+ "stages": [
12
+ {
13
+ "routes": [
14
+ "knowledge_qa"
15
+ ],
16
+ "authentic": "single"
17
+ },
18
+ {
19
+ "routes": [
20
+ "reasoning_qa",
21
+ "stem_reasoning",
22
+ "how_to_qa",
23
+ "opinion_qa",
24
+ "composition_qa",
25
+ "verse_qa"
26
+ ],
27
+ "calibration_qa": true
28
+ },
29
+ {
30
+ "routes": [
31
+ "multiturn_qa",
32
+ "narrative_grounded",
33
+ "narrative_fiction"
34
+ ],
35
+ "authentic": "multi"
36
+ }
37
+ ],
38
+ "noise": {
39
+ "rate": 0.3
40
+ },
41
+ "robustness": {
42
+ "stage": 0,
43
+ "epochs": 1,
44
+ "routes": {
45
+ "conversation_qa": {
46
+ "count": null
47
+ },
48
+ "unparseable_qa": {
49
+ "count": null
50
+ },
51
+ "typo_qa": {
52
+ "count": null
53
+ },
54
+ "era_qa": {
55
+ "count": null
56
+ },
57
+ "conversation_multiturn": {
58
+ "count": null
59
+ }
60
+ }
61
+ }
62
+ }
63
+ },
64
+ "training": {
65
+ "num_iterations": -1,
66
+ "load_optimizer": 0,
67
+ "device_batch_size": 2,
68
+ "warmup_ratio": 0.03,
69
+ "eval_every": 400,
70
+ "chatcore_every": 100000,
71
+ "chatcore_max_sample": 32,
72
+ "save_every": -1
73
+ },
74
+ "artifacts": {
75
+ "repo": "jbduran/think.nano"
76
+ },
77
+ "wandb": {
78
+ "enabled": true,
79
+ "entity": "jbduran-thinkingmachinesncsu",
80
+ "project": "think.nano",
81
+ "group": "think-d32",
82
+ "tags": [
83
+ "sft",
84
+ "curriculum",
85
+ "c3",
86
+ "scale-max",
87
+ "staged",
88
+ "robustness",
89
+ "noise",
90
+ "d32"
91
+ ]
92
+ },
93
+ "config_fingerprint": "4462a389dc98cd5d",
94
+ "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2"
95
+ }
evals/chatcore.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "stage": "sft",
3
+ "step": 42,
4
+ "total_training_flops": 2.3204581023414642e+20,
5
+ "stage_training_flops": 1.0107782648656036e+18,
6
+ "inherited_parent_flops": 2.3103503196928082e+20,
7
+ "cumulative_pipeline_training_flops": 2.3204581023414642e+20,
8
+ "results": {
9
+ "ARC-Easy": 0.24579124579124578,
10
+ "ARC-Challenge": 0.23037542662116042,
11
+ "MMLU": 0.23137729668138443,
12
+ "GSM8K": 0.0,
13
+ "HumanEval": 0.0
14
+ },
15
+ "chatcore_metric": -0.011321608241655831,
16
+ "chatcore_suite": {
17
+ "name": "karpathy",
18
+ "tasks": [
19
+ "ARC-Easy",
20
+ "ARC-Challenge",
21
+ "MMLU",
22
+ "GSM8K",
23
+ "HumanEval"
24
+ ],
25
+ "max_generative_problems": null,
26
+ "generative_answer_format": null
27
+ },
28
+ "complete": true
29
+ }
run.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2",
3
+ "stage": "sft",
4
+ "base_experiment_id": "Think.Unbounded-d32-v2mix-cont",
5
+ "parent_experiment_id": "Think.Unbounded-d32-v2mix-cont",
6
+ "parent_checkpoint_step": 9600,
7
+ "branch_parent_step": null,
8
+ "config_fingerprint": "4462a389dc98cd5d",
9
+ "wandb_run_id": "036b1c71",
10
+ "created_at": 1786715992
11
+ }