Hugging Face's logo

hellosindh
/
sindhi-bert-base

Safetensors
Sindhi
roberta
sindhi
bert
masked-language-modeling
from-scratch
Model card Files
xet
 Community
hellosindh commited on
Commit
e085157
·
verified ·
1 Parent(s): 0f90e34

Upload folder using huggingface_hub

Browse files
Files changed (17) hide show
  1. README.md +7 -7
  2. checkpoint-3261/config.json +28 -0
  3. checkpoint-3261/model.safetensors +3 -0
  4. checkpoint-3261/optimizer.pt +3 -0
  5. checkpoint-3261/rng_state.pth +3 -0
  6. checkpoint-3261/scheduler.pt +3 -0
  7. checkpoint-3261/trainer_state.json +266 -0
  8. checkpoint-3261/training_args.bin +3 -0
  9. checkpoint-6522/config.json +28 -0
  10. checkpoint-6522/model.safetensors +3 -0
  11. checkpoint-6522/optimizer.pt +3 -0
  12. checkpoint-6522/rng_state.pth +3 -0
  13. checkpoint-6522/scheduler.pt +3 -0
  14. checkpoint-6522/trainer_state.json +505 -0
  15. checkpoint-6522/training_args.bin +3 -0
  16. model.safetensors +1 -1
  17. training_args.bin +2 -2
README.md CHANGED
@@ -17,12 +17,12 @@ First BERT-style model trained from scratch on Sindhi text.
17
 
18
  | Session | Data | Epochs | PPL | Notes |
19
  |---|---|---|---|---|
20
- | S1 | 500K lines | 5 | 78.10 | from scratch |
21
- | S2 | 1.5M lines | 3 | 41.62 | continued |
22
- | S3 | 87M words | 2 | 28.46 | bf16, cosine LR |
23
- | S4 | 87M words | 3 | 35.16 | grouped context, MLM=0.20 |
24
- | S5 | 87M words | 2 | 29.67 | fine polish, MLM=0.15 |
25
- | S6 | 149M words | 2 | 32.76 | ultra fine, LR=3e-6, group=100 |
26
 
27
  ## Usage
28
 
@@ -32,7 +32,7 @@ import sentencepiece as spm, torch
32
  import torch.nn.functional as F
33
  from huggingface_hub import hf_hub_download
34
 
35
- REPO = "hellosindh/sindhi-bert-base"
36
  MASK_ID = 32000
37
  BOS_ID = 2
38
  EOS_ID = 3
 
17
 
18
  | Session | Data | Epochs | PPL | Notes |
19
  |---|---|---|---|---|
20
+ | S1 | 500K lines | 5 | 78.10 | from scratch |
21
+ | S2 | 1.5M lines | 3 | 41.62 | continued |
22
+ | S3 | 87M words | 2 | 28.46 | bf16, cosine LR |
23
+ | S4 | 87M words | 3 | 35.16 | grouped context, MLM=0.20 |
24
+ | S5 | 87M words | 2 | 29.67 | fine polish, MLM=0.15 |
25
+ | S6r | 149M words | 2 | 31.66 | grouping=80, LR=5e-6 |
26
 
27
  ## Usage
28
 
 
32
  import torch.nn.functional as F
33
  from huggingface_hub import hf_hub_download
34
 
35
+ REPO = "hellosindh/sindhi-bert-base"
36
  MASK_ID = 32000
37
  BOS_ID = 2
38
  EOS_ID = 3
checkpoint-3261/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "RobertaForMaskedLM"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": 1,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": 2,
11
+ "hidden_act": "gelu",
12
+ "hidden_dropout_prob": 0.1,
13
+ "hidden_size": 768,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 3072,
16
+ "is_decoder": false,
17
+ "layer_norm_eps": 1e-12,
18
+ "max_position_embeddings": 514,
19
+ "model_type": "roberta",
20
+ "num_attention_heads": 12,
21
+ "num_hidden_layers": 12,
22
+ "pad_token_id": 0,
23
+ "tie_word_embeddings": true,
24
+ "transformers_version": "5.0.0",
25
+ "type_vocab_size": 1,
26
+ "use_cache": false,
27
+ "vocab_size": 32001
28
+ }
checkpoint-3261/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:668b23d0a704e1fb45b068fdaa0de7644f146adc7e79903f3cbcbe733a2c6970
3
+ size 442633860
checkpoint-3261/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d8d158e7a892b99b2d1aa72b5bedc9f59bf1c5b06698c07f2dc784e6f028aa5
3
+ size 885391563
checkpoint-3261/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3c412beaf0b44d2d9fe547f685d72a840a5db54f9c041543d2096cc61a54a7d
3
+ size 14645
checkpoint-3261/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:389e1defe01003138f2ef356ab9b6b29de06fe1bd8acabe84c55f58b6f52567f
3
+ size 1465
checkpoint-3261/trainer_state.json ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 3261,
3
+ "best_metric": 3.45804762840271,
4
+ "best_model_checkpoint": "sindhibert_session6r/checkpoint-3261",
5
+ "epoch": 1.0,
6
+ "eval_steps": 3261,
7
+ "global_step": 3261,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.030672494440610383,
14
+ "grad_norm": 5.580650806427002,
15
+ "learning_rate": 1.2659846547314581e-06,
16
+ "loss": 14.62664794921875,
17
+ "step": 100
18
+ },
19
+ {
20
+ "epoch": 0.061344988881220766,
21
+ "grad_norm": 5.809163570404053,
22
+ "learning_rate": 2.544757033248082e-06,
23
+ "loss": 14.634923095703124,
24
+ "step": 200
25
+ },
26
+ {
27
+ "epoch": 0.09201748332183114,
28
+ "grad_norm": 5.849374294281006,
29
+ "learning_rate": 3.8235294117647055e-06,
30
+ "loss": 14.635355224609375,
31
+ "step": 300
32
+ },
33
+ {
34
+ "epoch": 0.12268997776244153,
35
+ "grad_norm": 5.751659870147705,
36
+ "learning_rate": 4.999978994815915e-06,
37
+ "loss": 14.59581298828125,
38
+ "step": 400
39
+ },
40
+ {
41
+ "epoch": 0.1533624722030519,
42
+ "grad_norm": 5.633106708526611,
43
+ "learning_rate": 4.996172776747675e-06,
44
+ "loss": 14.598475341796876,
45
+ "step": 500
46
+ },
47
+ {
48
+ "epoch": 0.18403496664366228,
49
+ "grad_norm": 5.498394012451172,
50
+ "learning_rate": 4.9858139123513936e-06,
51
+ "loss": 14.56541259765625,
52
+ "step": 600
53
+ },
54
+ {
55
+ "epoch": 0.2147074610842727,
56
+ "grad_norm": 5.918303489685059,
57
+ "learning_rate": 4.968929594446166e-06,
58
+ "loss": 14.57242919921875,
59
+ "step": 700
60
+ },
61
+ {
62
+ "epoch": 0.24537995552488306,
63
+ "grad_norm": 5.2229814529418945,
64
+ "learning_rate": 4.945564145670534e-06,
65
+ "loss": 14.53744873046875,
66
+ "step": 800
67
+ },
68
+ {
69
+ "epoch": 0.27605244996549344,
70
+ "grad_norm": 5.516529560089111,
71
+ "learning_rate": 4.9157789021321375e-06,
72
+ "loss": 14.531802978515625,
73
+ "step": 900
74
+ },
75
+ {
76
+ "epoch": 0.3067249444061038,
77
+ "grad_norm": 5.460958003997803,
78
+ "learning_rate": 4.879652052395696e-06,
79
+ "loss": 14.5186328125,
80
+ "step": 1000
81
+ },
82
+ {
83
+ "epoch": 0.3373974388467142,
84
+ "grad_norm": 5.609668731689453,
85
+ "learning_rate": 4.837278432231982e-06,
86
+ "loss": 14.51820068359375,
87
+ "step": 1100
88
+ },
89
+ {
90
+ "epoch": 0.36806993328732457,
91
+ "grad_norm": 5.466991424560547,
92
+ "learning_rate": 4.788769275666605e-06,
93
+ "loss": 14.477803955078125,
94
+ "step": 1200
95
+ },
96
+ {
97
+ "epoch": 0.398742427727935,
98
+ "grad_norm": 5.586793899536133,
99
+ "learning_rate": 4.7342519229820996e-06,
100
+ "loss": 14.506885986328125,
101
+ "step": 1300
102
+ },
103
+ {
104
+ "epoch": 0.4294149221685454,
105
+ "grad_norm": 5.488733768463135,
106
+ "learning_rate": 4.673869486439846e-06,
107
+ "loss": 14.464423828125,
108
+ "step": 1400
109
+ },
110
+ {
111
+ "epoch": 0.46008741660915575,
112
+ "grad_norm": 5.8738017082214355,
113
+ "learning_rate": 4.60778047459934e-06,
114
+ "loss": 14.470579833984376,
115
+ "step": 1500
116
+ },
117
+ {
118
+ "epoch": 0.49075991104976613,
119
+ "grad_norm": 5.592001438140869,
120
+ "learning_rate": 4.536158376220971e-06,
121
+ "loss": 14.42775146484375,
122
+ "step": 1600
123
+ },
124
+ {
125
+ "epoch": 0.5214324054903765,
126
+ "grad_norm": 5.235401153564453,
127
+ "learning_rate": 4.459191204844634e-06,
128
+ "loss": 14.42791259765625,
129
+ "step": 1700
130
+ },
131
+ {
132
+ "epoch": 0.5521048999309869,
133
+ "grad_norm": 5.959557056427002,
134
+ "learning_rate": 4.377081005239665e-06,
135
+ "loss": 14.444658203125,
136
+ "step": 1800
137
+ },
138
+ {
139
+ "epoch": 0.5827773943715973,
140
+ "grad_norm": 5.362053394317627,
141
+ "learning_rate": 4.2900433230217155e-06,
142
+ "loss": 14.463519287109374,
143
+ "step": 1900
144
+ },
145
+ {
146
+ "epoch": 0.6134498888122076,
147
+ "grad_norm": 5.445187568664551,
148
+ "learning_rate": 4.198306638828863e-06,
149
+ "loss": 14.472896728515625,
150
+ "step": 2000
151
+ },
152
+ {
153
+ "epoch": 0.644122383252818,
154
+ "grad_norm": 5.700764179229736,
155
+ "learning_rate": 4.102111768542294e-06,
156
+ "loss": 14.44870849609375,
157
+ "step": 2100
158
+ },
159
+ {
160
+ "epoch": 0.6747948776934284,
161
+ "grad_norm": 5.197051525115967,
162
+ "learning_rate": 4.001711231126012e-06,
163
+ "loss": 14.42983642578125,
164
+ "step": 2200
165
+ },
166
+ {
167
+ "epoch": 0.7054673721340388,
168
+ "grad_norm": 5.519675254821777,
169
+ "learning_rate": 3.897368585745059e-06,
170
+ "loss": 14.4178271484375,
171
+ "step": 2300
172
+ },
173
+ {
174
+ "epoch": 0.7361398665746491,
175
+ "grad_norm": 5.8170647621154785,
176
+ "learning_rate": 3.789357739902342e-06,
177
+ "loss": 14.38358642578125,
178
+ "step": 2400
179
+ },
180
+ {
181
+ "epoch": 0.7668123610152595,
182
+ "grad_norm": 5.513352394104004,
183
+ "learning_rate": 3.6779622304102957e-06,
184
+ "loss": 14.41911376953125,
185
+ "step": 2500
186
+ },
187
+ {
188
+ "epoch": 0.79748485545587,
189
+ "grad_norm": 5.644667625427246,
190
+ "learning_rate": 3.5634744790848436e-06,
191
+ "loss": 14.40310791015625,
192
+ "step": 2600
193
+ },
194
+ {
195
+ "epoch": 0.8281573498964804,
196
+ "grad_norm": 5.676363945007324,
197
+ "learning_rate": 3.4461950251155457e-06,
198
+ "loss": 14.40308349609375,
199
+ "step": 2700
200
+ },
201
+ {
202
+ "epoch": 0.8588298443370908,
203
+ "grad_norm": 5.251427173614502,
204
+ "learning_rate": 3.326431736127017e-06,
205
+ "loss": 14.39386474609375,
206
+ "step": 2800
207
+ },
208
+ {
209
+ "epoch": 0.8895023387777011,
210
+ "grad_norm": 5.631777286529541,
211
+ "learning_rate": 3.2044990000026366e-06,
212
+ "loss": 14.374044189453125,
213
+ "step": 2900
214
+ },
215
+ {
216
+ "epoch": 0.9201748332183115,
217
+ "grad_norm": 5.5782670974731445,
218
+ "learning_rate": 3.080716899592065e-06,
219
+ "loss": 14.37978515625,
220
+ "step": 3000
221
+ },
222
+ {
223
+ "epoch": 0.9508473276589219,
224
+ "grad_norm": 5.784708023071289,
225
+ "learning_rate": 2.9554103724690526e-06,
226
+ "loss": 14.3670703125,
227
+ "step": 3100
228
+ },
229
+ {
230
+ "epoch": 0.9815198220995323,
231
+ "grad_norm": 5.501222610473633,
232
+ "learning_rate": 2.8289083579452043e-06,
233
+ "loss": 14.374014892578124,
234
+ "step": 3200
235
+ },
236
+ {
237
+ "epoch": 1.0,
238
+ "eval_loss": 3.45804762840271,
239
+ "eval_runtime": 13.1436,
240
+ "eval_samples_per_second": 641.376,
241
+ "eval_steps_per_second": 10.043,
242
+ "step": 3261
243
+ }
244
+ ],
245
+ "logging_steps": 100,
246
+ "max_steps": 6522,
247
+ "num_input_tokens_seen": 0,
248
+ "num_train_epochs": 2,
249
+ "save_steps": 3261,
250
+ "stateful_callbacks": {
251
+ "TrainerControl": {
252
+ "args": {
253
+ "should_epoch_stop": false,
254
+ "should_evaluate": false,
255
+ "should_log": false,
256
+ "should_save": true,
257
+ "should_training_stop": false
258
+ },
259
+ "attributes": {}
260
+ }
261
+ },
262
+ "total_flos": 2.1967584480474624e+17,
263
+ "train_batch_size": 64,
264
+ "trial_name": null,
265
+ "trial_params": null
266
+ }
checkpoint-3261/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73e3a1459dd5072b21a8551a6be8b64cb13b36dccfa9aaf716a69eda69a1ca83
3
+ size 5201
checkpoint-6522/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "RobertaForMaskedLM"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": 1,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": 2,
11
+ "hidden_act": "gelu",
12
+ "hidden_dropout_prob": 0.1,
13
+ "hidden_size": 768,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 3072,
16
+ "is_decoder": false,
17
+ "layer_norm_eps": 1e-12,
18
+ "max_position_embeddings": 514,
19
+ "model_type": "roberta",
20
+ "num_attention_heads": 12,
21
+ "num_hidden_layers": 12,
22
+ "pad_token_id": 0,
23
+ "tie_word_embeddings": true,
24
+ "transformers_version": "5.0.0",
25
+ "type_vocab_size": 1,
26
+ "use_cache": false,
27
+ "vocab_size": 32001
28
+ }
checkpoint-6522/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:145f5d33e2971984e7d3b3e1f6d5180ed698917b83dd97f4eeb4c5459b46d18e
3
+ size 442633860
checkpoint-6522/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dda22037182dd7b9ad54a494c0a12243bea69a36b59497840af05e8522af8df
3
+ size 885391563
checkpoint-6522/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9257c59c1a3b91e421df3a2b4bfb4ef1cc980e5fd1e4f947b129a78116566786
3
+ size 14645
checkpoint-6522/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69ee71dcc00bf9f208b41aa1b8f5a0daf7da45f56e13e0631d8baaaafbe2ffe4
3
+ size 1465
checkpoint-6522/trainer_state.json ADDED
@@ -0,0 +1,505 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 6522,
3
+ "best_metric": 3.4418444633483887,
4
+ "best_model_checkpoint": "sindhibert_session6r/checkpoint-6522",
5
+ "epoch": 2.0,
6
+ "eval_steps": 3261,
7
+ "global_step": 6522,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.030672494440610383,
14
+ "grad_norm": 5.580650806427002,
15
+ "learning_rate": 1.2659846547314581e-06,
16
+ "loss": 14.62664794921875,
17
+ "step": 100
18
+ },
19
+ {
20
+ "epoch": 0.061344988881220766,
21
+ "grad_norm": 5.809163570404053,
22
+ "learning_rate": 2.544757033248082e-06,
23
+ "loss": 14.634923095703124,
24
+ "step": 200
25
+ },
26
+ {
27
+ "epoch": 0.09201748332183114,
28
+ "grad_norm": 5.849374294281006,
29
+ "learning_rate": 3.8235294117647055e-06,
30
+ "loss": 14.635355224609375,
31
+ "step": 300
32
+ },
33
+ {
34
+ "epoch": 0.12268997776244153,
35
+ "grad_norm": 5.751659870147705,
36
+ "learning_rate": 4.999978994815915e-06,
37
+ "loss": 14.59581298828125,
38
+ "step": 400
39
+ },
40
+ {
41
+ "epoch": 0.1533624722030519,
42
+ "grad_norm": 5.633106708526611,
43
+ "learning_rate": 4.996172776747675e-06,
44
+ "loss": 14.598475341796876,
45
+ "step": 500
46
+ },
47
+ {
48
+ "epoch": 0.18403496664366228,
49
+ "grad_norm": 5.498394012451172,
50
+ "learning_rate": 4.9858139123513936e-06,
51
+ "loss": 14.56541259765625,
52
+ "step": 600
53
+ },
54
+ {
55
+ "epoch": 0.2147074610842727,
56
+ "grad_norm": 5.918303489685059,
57
+ "learning_rate": 4.968929594446166e-06,
58
+ "loss": 14.57242919921875,
59
+ "step": 700
60
+ },
61
+ {
62
+ "epoch": 0.24537995552488306,
63
+ "grad_norm": 5.2229814529418945,
64
+ "learning_rate": 4.945564145670534e-06,
65
+ "loss": 14.53744873046875,
66
+ "step": 800
67
+ },
68
+ {
69
+ "epoch": 0.27605244996549344,
70
+ "grad_norm": 5.516529560089111,
71
+ "learning_rate": 4.9157789021321375e-06,
72
+ "loss": 14.531802978515625,
73
+ "step": 900
74
+ },
75
+ {
76
+ "epoch": 0.3067249444061038,
77
+ "grad_norm": 5.460958003997803,
78
+ "learning_rate": 4.879652052395696e-06,
79
+ "loss": 14.5186328125,
80
+ "step": 1000
81
+ },
82
+ {
83
+ "epoch": 0.3373974388467142,
84
+ "grad_norm": 5.609668731689453,
85
+ "learning_rate": 4.837278432231982e-06,
86
+ "loss": 14.51820068359375,
87
+ "step": 1100
88
+ },
89
+ {
90
+ "epoch": 0.36806993328732457,
91
+ "grad_norm": 5.466991424560547,
92
+ "learning_rate": 4.788769275666605e-06,
93
+ "loss": 14.477803955078125,
94
+ "step": 1200
95
+ },
96
+ {
97
+ "epoch": 0.398742427727935,
98
+ "grad_norm": 5.586793899536133,
99
+ "learning_rate": 4.7342519229820996e-06,
100
+ "loss": 14.506885986328125,
101
+ "step": 1300
102
+ },
103
+ {
104
+ "epoch": 0.4294149221685454,
105
+ "grad_norm": 5.488733768463135,
106
+ "learning_rate": 4.673869486439846e-06,
107
+ "loss": 14.464423828125,
108
+ "step": 1400
109
+ },
110
+ {
111
+ "epoch": 0.46008741660915575,
112
+ "grad_norm": 5.8738017082214355,
113
+ "learning_rate": 4.60778047459934e-06,
114
+ "loss": 14.470579833984376,
115
+ "step": 1500
116
+ },
117
+ {
118
+ "epoch": 0.49075991104976613,
119
+ "grad_norm": 5.592001438140869,
120
+ "learning_rate": 4.536158376220971e-06,
121
+ "loss": 14.42775146484375,
122
+ "step": 1600
123
+ },
124
+ {
125
+ "epoch": 0.5214324054903765,
126
+ "grad_norm": 5.235401153564453,
127
+ "learning_rate": 4.459191204844634e-06,
128
+ "loss": 14.42791259765625,
129
+ "step": 1700
130
+ },
131
+ {
132
+ "epoch": 0.5521048999309869,
133
+ "grad_norm": 5.959557056427002,
134
+ "learning_rate": 4.377081005239665e-06,
135
+ "loss": 14.444658203125,
136
+ "step": 1800
137
+ },
138
+ {
139
+ "epoch": 0.5827773943715973,
140
+ "grad_norm": 5.362053394317627,
141
+ "learning_rate": 4.2900433230217155e-06,
142
+ "loss": 14.463519287109374,
143
+ "step": 1900
144
+ },
145
+ {
146
+ "epoch": 0.6134498888122076,
147
+ "grad_norm": 5.445187568664551,
148
+ "learning_rate": 4.198306638828863e-06,
149
+ "loss": 14.472896728515625,
150
+ "step": 2000
151
+ },
152
+ {
153
+ "epoch": 0.644122383252818,
154
+ "grad_norm": 5.700764179229736,
155
+ "learning_rate": 4.102111768542294e-06,
156
+ "loss": 14.44870849609375,
157
+ "step": 2100
158
+ },
159
+ {
160
+ "epoch": 0.6747948776934284,
161
+ "grad_norm": 5.197051525115967,
162
+ "learning_rate": 4.001711231126012e-06,
163
+ "loss": 14.42983642578125,
164
+ "step": 2200
165
+ },
166
+ {
167
+ "epoch": 0.7054673721340388,
168
+ "grad_norm": 5.519675254821777,
169
+ "learning_rate": 3.897368585745059e-06,
170
+ "loss": 14.4178271484375,
171
+ "step": 2300
172
+ },
173
+ {
174
+ "epoch": 0.7361398665746491,
175
+ "grad_norm": 5.8170647621154785,
176
+ "learning_rate": 3.789357739902342e-06,
177
+ "loss": 14.38358642578125,
178
+ "step": 2400
179
+ },
180
+ {
181
+ "epoch": 0.7668123610152595,
182
+ "grad_norm": 5.513352394104004,
183
+ "learning_rate": 3.6779622304102957e-06,
184
+ "loss": 14.41911376953125,
185
+ "step": 2500
186
+ },
187
+ {
188
+ "epoch": 0.79748485545587,
189
+ "grad_norm": 5.644667625427246,
190
+ "learning_rate": 3.5634744790848436e-06,
191
+ "loss": 14.40310791015625,
192
+ "step": 2600
193
+ },
194
+ {
195
+ "epoch": 0.8281573498964804,
196
+ "grad_norm": 5.676363945007324,
197
+ "learning_rate": 3.4461950251155457e-06,
198
+ "loss": 14.40308349609375,
199
+ "step": 2700
200
+ },
201
+ {
202
+ "epoch": 0.8588298443370908,
203
+ "grad_norm": 5.251427173614502,
204
+ "learning_rate": 3.326431736127017e-06,
205
+ "loss": 14.39386474609375,
206
+ "step": 2800
207
+ },
208
+ {
209
+ "epoch": 0.8895023387777011,
210
+ "grad_norm": 5.631777286529541,
211
+ "learning_rate": 3.2044990000026366e-06,
212
+ "loss": 14.374044189453125,
213
+ "step": 2900
214
+ },
215
+ {
216
+ "epoch": 0.9201748332183115,
217
+ "grad_norm": 5.5782670974731445,
218
+ "learning_rate": 3.080716899592065e-06,
219
+ "loss": 14.37978515625,
220
+ "step": 3000
221
+ },
222
+ {
223
+ "epoch": 0.9508473276589219,
224
+ "grad_norm": 5.784708023071289,
225
+ "learning_rate": 2.9554103724690526e-06,
226
+ "loss": 14.3670703125,
227
+ "step": 3100
228
+ },
229
+ {
230
+ "epoch": 0.9815198220995323,
231
+ "grad_norm": 5.501222610473633,
232
+ "learning_rate": 2.8289083579452043e-06,
233
+ "loss": 14.374014892578124,
234
+ "step": 3200
235
+ },
236
+ {
237
+ "epoch": 1.0,
238
+ "eval_loss": 3.45804762840271,
239
+ "eval_runtime": 13.1436,
240
+ "eval_samples_per_second": 641.376,
241
+ "eval_steps_per_second": 10.043,
242
+ "step": 3261
243
+ },
244
+ {
245
+ "epoch": 1.011962272831838,
246
+ "grad_norm": 5.622377872467041,
247
+ "learning_rate": 2.7015429335788844e-06,
248
+ "loss": 14.245347900390625,
249
+ "step": 3300
250
+ },
251
+ {
252
+ "epoch": 1.0426347672724485,
253
+ "grad_norm": 5.506319522857666,
254
+ "learning_rate": 2.57364844344598e-06,
255
+ "loss": 14.36246826171875,
256
+ "step": 3400
257
+ },
258
+ {
259
+ "epoch": 1.0733072617130588,
260
+ "grad_norm": 5.29400634765625,
261
+ "learning_rate": 2.4455606204608764e-06,
262
+ "loss": 14.3725537109375,
263
+ "step": 3500
264
+ },
265
+ {
266
+ "epoch": 1.1039797561536693,
267
+ "grad_norm": 5.597475528717041,
268
+ "learning_rate": 2.3176157050516264e-06,
269
+ "loss": 14.357423095703124,
270
+ "step": 3600
271
+ },
272
+ {
273
+ "epoch": 1.1346522505942795,
274
+ "grad_norm": 5.25772762298584,
275
+ "learning_rate": 2.1901495625028606e-06,
276
+ "loss": 14.315875244140624,
277
+ "step": 3700
278
+ },
279
+ {
280
+ "epoch": 1.16532474503489,
281
+ "grad_norm": 5.585110187530518,
282
+ "learning_rate": 2.063496801283472e-06,
283
+ "loss": 14.327174072265626,
284
+ "step": 3800
285
+ },
286
+ {
287
+ "epoch": 1.1959972394755003,
288
+ "grad_norm": 5.402249813079834,
289
+ "learning_rate": 1.9379898946735452e-06,
290
+ "loss": 14.34656494140625,
291
+ "step": 3900
292
+ },
293
+ {
294
+ "epoch": 1.2266697339161108,
295
+ "grad_norm": 5.390650749206543,
296
+ "learning_rate": 1.8139583079963144e-06,
297
+ "loss": 14.36371826171875,
298
+ "step": 4000
299
+ },
300
+ {
301
+ "epoch": 1.257342228356721,
302
+ "grad_norm": 5.242497444152832,
303
+ "learning_rate": 1.6917276337462466e-06,
304
+ "loss": 14.323082275390625,
305
+ "step": 4100
306
+ },
307
+ {
308
+ "epoch": 1.2880147227973315,
309
+ "grad_norm": 5.234506130218506,
310
+ "learning_rate": 1.5716187368835971e-06,
311
+ "loss": 14.337457275390625,
312
+ "step": 4200
313
+ },
314
+ {
315
+ "epoch": 1.3186872172379418,
316
+ "grad_norm": 5.626885890960693,
317
+ "learning_rate": 1.4539469125391031e-06,
318
+ "loss": 14.354864501953125,
319
+ "step": 4300
320
+ },
321
+ {
322
+ "epoch": 1.3493597116785523,
323
+ "grad_norm": 5.311978816986084,
324
+ "learning_rate": 1.3390210583399215e-06,
325
+ "loss": 14.32729248046875,
326
+ "step": 4400
327
+ },
328
+ {
329
+ "epoch": 1.3800322061191626,
330
+ "grad_norm": 5.375194072723389,
331
+ "learning_rate": 1.2271428635294922e-06,
332
+ "loss": 14.337626953125,
333
+ "step": 4500
334
+ },
335
+ {
336
+ "epoch": 1.410704700559773,
337
+ "grad_norm": 5.57642936706543,
338
+ "learning_rate": 1.118606017009978e-06,
339
+ "loss": 14.334080810546874,
340
+ "step": 4600
341
+ },
342
+ {
343
+ "epoch": 1.4413771950003835,
344
+ "grad_norm": 5.510963439941406,
345
+ "learning_rate": 1.0136954363862126e-06,
346
+ "loss": 14.33389404296875,
347
+ "step": 4700
348
+ },
349
+ {
350
+ "epoch": 1.4720496894409938,
351
+ "grad_norm": 5.4500041007995605,
352
+ "learning_rate": 9.126865200349847e-07,
353
+ "loss": 14.31758056640625,
354
+ "step": 4800
355
+ },
356
+ {
357
+ "epoch": 1.502722183881604,
358
+ "grad_norm": 5.399460792541504,
359
+ "learning_rate": 8.158444241630245e-07,
360
+ "loss": 14.280606689453125,
361
+ "step": 4900
362
+ },
363
+ {
364
+ "epoch": 1.5333946783222145,
365
+ "grad_norm": 5.639800071716309,
366
+ "learning_rate": 7.234233667514868e-07,
367
+ "loss": 14.342408447265624,
368
+ "step": 5000
369
+ },
370
+ {
371
+ "epoch": 1.564067172762825,
372
+ "grad_norm": 5.429558277130127,
373
+ "learning_rate": 6.356659602141116e-07,
374
+ "loss": 14.314246826171875,
375
+ "step": 5100
376
+ },
377
+ {
378
+ "epoch": 1.5947396672034353,
379
+ "grad_norm": 5.131410598754883,
380
+ "learning_rate": 5.528025745209039e-07,
381
+ "loss": 14.32427978515625,
382
+ "step": 5200
383
+ },
384
+ {
385
+ "epoch": 1.6254121616440456,
386
+ "grad_norm": 5.480471611022949,
387
+ "learning_rate": 4.7505073245916774e-07,
388
+ "loss": 14.32928466796875,
389
+ "step": 5300
390
+ },
391
+ {
392
+ "epoch": 1.656084656084656,
393
+ "grad_norm": 5.693525791168213,
394
+ "learning_rate": 4.026145386193914e-07,
395
+ "loss": 14.34359375,
396
+ "step": 5400
397
+ },
398
+ {
399
+ "epoch": 1.6867571505252665,
400
+ "grad_norm": 5.723335266113281,
401
+ "learning_rate": 3.3568414360493075e-07,
402
+ "loss": 14.31550537109375,
403
+ "step": 5500
404
+ },
405
+ {
406
+ "epoch": 1.7174296449658768,
407
+ "grad_norm": 5.319007873535156,
408
+ "learning_rate": 2.7443524487199333e-07,
409
+ "loss": 14.32018310546875,
410
+ "step": 5600
411
+ },
412
+ {
413
+ "epoch": 1.7481021394064873,
414
+ "grad_norm": 5.462660312652588,
415
+ "learning_rate": 2.1902862551024112e-07,
416
+ "loss": 14.333775634765624,
417
+ "step": 5700
418
+ },
419
+ {
420
+ "epoch": 1.7787746338470978,
421
+ "grad_norm": 5.3777546882629395,
422
+ "learning_rate": 1.6960973217476778e-07,
423
+ "loss": 14.32505126953125,
424
+ "step": 5800
425
+ },
426
+ {
427
+ "epoch": 1.809447128287708,
428
+ "grad_norm": 5.701692581176758,
429
+ "learning_rate": 1.2630829327738987e-07,
430
+ "loss": 14.33883056640625,
431
+ "step": 5900
432
+ },
433
+ {
434
+ "epoch": 1.8401196227283183,
435
+ "grad_norm": 5.285277843475342,
436
+ "learning_rate": 8.92379784395514e-08,
437
+ "loss": 14.3292626953125,
438
+ "step": 6000
439
+ },
440
+ {
441
+ "epoch": 1.8707921171689288,
442
+ "grad_norm": 5.558449745178223,
443
+ "learning_rate": 5.849610010078499e-08,
444
+ "loss": 14.3488134765625,
445
+ "step": 6100
446
+ },
447
+ {
448
+ "epoch": 1.9014646116095393,
449
+ "grad_norm": 5.662689685821533,
450
+ "learning_rate": 3.416335806604443e-08,
451
+ "loss": 14.28878173828125,
452
+ "step": 6200
453
+ },
454
+ {
455
+ "epoch": 1.9321371060501495,
456
+ "grad_norm": 5.507394790649414,
457
+ "learning_rate": 1.630362766248256e-08,
458
+ "loss": 14.3447216796875,
459
+ "step": 6300
460
+ },
461
+ {
462
+ "epoch": 1.9628096004907598,
463
+ "grad_norm": 5.682081699371338,
464
+ "learning_rate": 4.963792061784822e-09,
465
+ "loss": 14.32853515625,
466
+ "step": 6400
467
+ },
468
+ {
469
+ "epoch": 1.9934820949313703,
470
+ "grad_norm": 5.3612589836120605,
471
+ "learning_rate": 1.736192082227306e-10,
472
+ "loss": 14.27859130859375,
473
+ "step": 6500
474
+ },
475
+ {
476
+ "epoch": 2.0,
477
+ "eval_loss": 3.4418444633483887,
478
+ "eval_runtime": 13.1653,
479
+ "eval_samples_per_second": 640.317,
480
+ "eval_steps_per_second": 10.026,
481
+ "step": 6522
482
+ }
483
+ ],
484
+ "logging_steps": 100,
485
+ "max_steps": 6522,
486
+ "num_input_tokens_seen": 0,
487
+ "num_train_epochs": 2,
488
+ "save_steps": 3261,
489
+ "stateful_callbacks": {
490
+ "TrainerControl": {
491
+ "args": {
492
+ "should_epoch_stop": false,
493
+ "should_evaluate": false,
494
+ "should_log": false,
495
+ "should_save": true,
496
+ "should_training_stop": true
497
+ },
498
+ "attributes": {}
499
+ }
500
+ },
501
+ "total_flos": 4.393516896094925e+17,
502
+ "train_batch_size": 64,
503
+ "trial_name": null,
504
+ "trial_params": null
505
+ }
checkpoint-6522/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73e3a1459dd5072b21a8551a6be8b64cb13b36dccfa9aaf716a69eda69a1ca83
3
+ size 5201
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:68752d540c779d0abd4f2ec922d2cee00c47a2a66d63fbd1517f3f2ba6b90f73
3
  size 442633860
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:145f5d33e2971984e7d3b3e1f6d5180ed698917b83dd97f4eeb4c5459b46d18e
3
  size 442633860
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7765c9e92276c99a6c5186a42ac8dca5be5882797e153f8ee76ffcc8912c2d0b
3
- size 5137
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73e3a1459dd5072b21a8551a6be8b64cb13b36dccfa9aaf716a69eda69a1ca83
3
+ size 5201