aixk commited on
Commit
9fcf5ae
·
verified ·
1 Parent(s): 739dfa4

Upload folder using huggingface_hub

Browse files
checkpoint-300/config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "FastHyperNet125mForCausalLM"
4
+ ],
5
+ "dtype": "float32",
6
+ "hidden_size": 768,
7
+ "hyperbolic_dim": 16,
8
+ "initializer_range": 0.02,
9
+ "intermediate_size": 2048,
10
+ "max_position_embeddings": 512,
11
+ "model_type": "fasthypernet_125m_tagless",
12
+ "num_attention_heads": 12,
13
+ "num_hidden_layers": 16,
14
+ "transformers_version": "5.13.1",
15
+ "use_cache": false,
16
+ "vocab_size": 8192
17
+ }
checkpoint-300/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "output_attentions": false,
4
+ "output_hidden_states": false,
5
+ "transformers_version": "5.13.1"
6
+ }
checkpoint-300/hybrid_tokenizer_mapping.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-300/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0545b7ba5f62acdc8e942de96284023ad71f8822c132a0566077221fdaffb3cb
3
+ size 501863088
checkpoint-300/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:60b1f51e51a1d016ab55de05b7cc5d85725b30fabaa682ed4ca8bdf200a53985
3
+ size 254046859
checkpoint-300/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d87d30bfdfa3afa36aa51e180aa0a5470961b7fbf6f682048f6db484c9d7039
3
+ size 14645
checkpoint-300/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2fa5948d3d1127fc7734c24851dfadb8b68652dc2ac26386fdae7f2387511c5
3
+ size 1383
checkpoint-300/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f08ef7c38cf5c6a064973ec387488522679e8640f5ffe556c1b8a99d9fbf95a
3
+ size 1465
checkpoint-300/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "HFCompatibleHybridTokenizer",
3
+ "vocab_size": 8192,
4
+ "model_max_length": 512,
5
+ "bos_token": "<bos>",
6
+ "eos_token": "<eos>",
7
+ "unk_token": "<unk>",
8
+ "pad_token": "<pad>"
9
+ }
checkpoint-300/trainer_state.json ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.004144090009635009,
6
+ "eval_steps": 500,
7
+ "global_step": 300,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.00013813633365450032,
14
+ "grad_norm": 1141.434814453125,
15
+ "learning_rate": 2.4e-07,
16
+ "loss": 10.0261,
17
+ "step": 10
18
+ },
19
+ {
20
+ "epoch": 0.00027627266730900063,
21
+ "grad_norm": 516.984375,
22
+ "learning_rate": 8.399999999999999e-07,
23
+ "loss": 9.839,
24
+ "step": 20
25
+ },
26
+ {
27
+ "epoch": 0.00041440900096350095,
28
+ "grad_norm": 471.4093933105469,
29
+ "learning_rate": 1.4399999999999998e-06,
30
+ "loss": 9.7314,
31
+ "step": 30
32
+ },
33
+ {
34
+ "epoch": 0.0005525453346180013,
35
+ "grad_norm": 712.4447631835938,
36
+ "learning_rate": 2.0399999999999995e-06,
37
+ "loss": 9.6854,
38
+ "step": 40
39
+ },
40
+ {
41
+ "epoch": 0.0006906816682725015,
42
+ "grad_norm": 486.08953857421875,
43
+ "learning_rate": 2.64e-06,
44
+ "loss": 9.6725,
45
+ "step": 50
46
+ },
47
+ {
48
+ "epoch": 0.0008288180019270019,
49
+ "grad_norm": 580.298828125,
50
+ "learning_rate": 3.24e-06,
51
+ "loss": 9.6168,
52
+ "step": 60
53
+ },
54
+ {
55
+ "epoch": 0.0009669543355815022,
56
+ "grad_norm": 463.0860290527344,
57
+ "learning_rate": 3.84e-06,
58
+ "loss": 9.5897,
59
+ "step": 70
60
+ },
61
+ {
62
+ "epoch": 0.0011050906692360025,
63
+ "grad_norm": 860.590576171875,
64
+ "learning_rate": 4.44e-06,
65
+ "loss": 9.5313,
66
+ "step": 80
67
+ },
68
+ {
69
+ "epoch": 0.0012432270028905027,
70
+ "grad_norm": 895.52685546875,
71
+ "learning_rate": 5.039999999999999e-06,
72
+ "loss": 9.4522,
73
+ "step": 90
74
+ },
75
+ {
76
+ "epoch": 0.001381363336545003,
77
+ "grad_norm": 643.9630737304688,
78
+ "learning_rate": 5.579999999999999e-06,
79
+ "loss": 9.4026,
80
+ "step": 100
81
+ },
82
+ {
83
+ "epoch": 0.0015194996701995034,
84
+ "grad_norm": 1676.3682861328125,
85
+ "learning_rate": 6.12e-06,
86
+ "loss": 9.2923,
87
+ "step": 110
88
+ },
89
+ {
90
+ "epoch": 0.0016576360038540038,
91
+ "grad_norm": 781.4235229492188,
92
+ "learning_rate": 6.719999999999999e-06,
93
+ "loss": 9.332,
94
+ "step": 120
95
+ },
96
+ {
97
+ "epoch": 0.001795772337508504,
98
+ "grad_norm": 2265.01318359375,
99
+ "learning_rate": 7.32e-06,
100
+ "loss": 9.2609,
101
+ "step": 130
102
+ },
103
+ {
104
+ "epoch": 0.0019339086711630043,
105
+ "grad_norm": 629.4778442382812,
106
+ "learning_rate": 7.919999999999999e-06,
107
+ "loss": 9.2692,
108
+ "step": 140
109
+ },
110
+ {
111
+ "epoch": 0.0020720450048175047,
112
+ "grad_norm": 625.36376953125,
113
+ "learning_rate": 8.52e-06,
114
+ "loss": 8.9624,
115
+ "step": 150
116
+ },
117
+ {
118
+ "epoch": 0.002210181338472005,
119
+ "grad_norm": 772.018310546875,
120
+ "learning_rate": 9.119999999999999e-06,
121
+ "loss": 8.9776,
122
+ "step": 160
123
+ },
124
+ {
125
+ "epoch": 0.0023483176721265054,
126
+ "grad_norm": 730.3453979492188,
127
+ "learning_rate": 9.719999999999998e-06,
128
+ "loss": 8.9601,
129
+ "step": 170
130
+ },
131
+ {
132
+ "epoch": 0.0024864540057810054,
133
+ "grad_norm": 879.9548950195312,
134
+ "learning_rate": 1.032e-05,
135
+ "loss": 8.8578,
136
+ "step": 180
137
+ },
138
+ {
139
+ "epoch": 0.0026245903394355057,
140
+ "grad_norm": 737.8306274414062,
141
+ "learning_rate": 1.0919999999999999e-05,
142
+ "loss": 8.9061,
143
+ "step": 190
144
+ },
145
+ {
146
+ "epoch": 0.002762726673090006,
147
+ "grad_norm": 1028.0711669921875,
148
+ "learning_rate": 1.1519999999999998e-05,
149
+ "loss": 8.8559,
150
+ "step": 200
151
+ },
152
+ {
153
+ "epoch": 0.0029008630067445065,
154
+ "grad_norm": 1324.770263671875,
155
+ "learning_rate": 1.212e-05,
156
+ "loss": 8.8061,
157
+ "step": 210
158
+ },
159
+ {
160
+ "epoch": 0.003038999340399007,
161
+ "grad_norm": 1034.25146484375,
162
+ "learning_rate": 1.2719999999999998e-05,
163
+ "loss": 8.6379,
164
+ "step": 220
165
+ },
166
+ {
167
+ "epoch": 0.0031771356740535072,
168
+ "grad_norm": 780.5751953125,
169
+ "learning_rate": 1.332e-05,
170
+ "loss": 8.6222,
171
+ "step": 230
172
+ },
173
+ {
174
+ "epoch": 0.0033152720077080076,
175
+ "grad_norm": 971.2122192382812,
176
+ "learning_rate": 1.3919999999999997e-05,
177
+ "loss": 8.5075,
178
+ "step": 240
179
+ },
180
+ {
181
+ "epoch": 0.0034534083413625075,
182
+ "grad_norm": 1695.25390625,
183
+ "learning_rate": 1.4519999999999998e-05,
184
+ "loss": 8.6544,
185
+ "step": 250
186
+ },
187
+ {
188
+ "epoch": 0.003591544675017008,
189
+ "grad_norm": 630.7172241210938,
190
+ "learning_rate": 1.512e-05,
191
+ "loss": 8.5996,
192
+ "step": 260
193
+ },
194
+ {
195
+ "epoch": 0.0037296810086715083,
196
+ "grad_norm": 405.32550048828125,
197
+ "learning_rate": 1.572e-05,
198
+ "loss": 8.4774,
199
+ "step": 270
200
+ },
201
+ {
202
+ "epoch": 0.0038678173423260086,
203
+ "grad_norm": 253.12026977539062,
204
+ "learning_rate": 1.6319999999999996e-05,
205
+ "loss": 8.2876,
206
+ "step": 280
207
+ },
208
+ {
209
+ "epoch": 0.004005953675980509,
210
+ "grad_norm": 691.5232543945312,
211
+ "learning_rate": 1.6919999999999997e-05,
212
+ "loss": 8.3093,
213
+ "step": 290
214
+ },
215
+ {
216
+ "epoch": 0.004144090009635009,
217
+ "grad_norm": 744.5304565429688,
218
+ "learning_rate": 1.7519999999999998e-05,
219
+ "loss": 8.4284,
220
+ "step": 300
221
+ }
222
+ ],
223
+ "logging_steps": 10,
224
+ "max_steps": 10000000,
225
+ "num_input_tokens_seen": 0,
226
+ "num_train_epochs": 139,
227
+ "save_steps": 100,
228
+ "stateful_callbacks": {
229
+ "TrainerControl": {
230
+ "args": {
231
+ "should_epoch_stop": false,
232
+ "should_evaluate": false,
233
+ "should_log": false,
234
+ "should_save": true,
235
+ "should_training_stop": false
236
+ },
237
+ "attributes": {}
238
+ }
239
+ },
240
+ "total_flos": 4563060144537600.0,
241
+ "train_batch_size": 8,
242
+ "trial_name": null,
243
+ "trial_params": null
244
+ }
checkpoint-300/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7abd0725d63733265c41cad05c967905e1477030408ea138ab9baf8646ee00d
3
+ size 5201