Hugging Face's logo

hellosindh
/
sindhi-bert-base

Safetensors
Sindhi
roberta
sindhi
bert
masked-language-modeling
from-scratch
Model card Files
xet
 Community
hellosindh commited on
Commit
0f90e34
·
verified ·
1 Parent(s): 49a2014

Upload folder using huggingface_hub

Browse files
Files changed (17) hide show
  1. README.md +3 -6
  2. checkpoint-2890/config.json +28 -0
  3. checkpoint-2890/model.safetensors +3 -0
  4. checkpoint-2890/optimizer.pt +3 -0
  5. checkpoint-2890/rng_state.pth +3 -0
  6. checkpoint-2890/scheduler.pt +3 -0
  7. checkpoint-2890/trainer_state.json +238 -0
  8. checkpoint-2890/training_args.bin +3 -0
  9. checkpoint-5780/config.json +28 -0
  10. checkpoint-5780/model.safetensors +3 -0
  11. checkpoint-5780/optimizer.pt +3 -0
  12. checkpoint-5780/rng_state.pth +3 -0
  13. checkpoint-5780/scheduler.pt +3 -0
  14. checkpoint-5780/trainer_state.json +449 -0
  15. checkpoint-5780/training_args.bin +3 -0
  16. model.safetensors +1 -1
  17. training_args.bin +1 -1
README.md CHANGED
@@ -6,11 +6,7 @@ tags:
6
  - sindhi
7
  - bert
8
  - masked-language-modeling
9
- - fill-mask
10
  - from-scratch
11
- pipeline_tag: fill-mask
12
- widget:
13
- - text: "پارس هڪ [MASK] آهي."
14
  ---
15
 
16
  # Sindhi-BERT-base
@@ -23,9 +19,10 @@ First BERT-style model trained from scratch on Sindhi text.
23
  |---|---|---|---|---|
24
  | S1 | 500K lines | 5 | 78.10 | from scratch |
25
  | S2 | 1.5M lines | 3 | 41.62 | continued |
26
- | S3 | 1.49M lines | 2 | 28.46 | bf16, cosine LR |
27
  | S4 | 87M words | 3 | 35.16 | grouped context, MLM=0.20 |
28
- | S5 | 87M words | 2 | 29.45 | fine polish, MLM=0.15 |
 
29
 
30
  ## Usage
31
 
 
6
  - sindhi
7
  - bert
8
  - masked-language-modeling
 
9
  - from-scratch
 
 
 
10
  ---
11
 
12
  # Sindhi-BERT-base
 
19
  |---|---|---|---|---|
20
  | S1 | 500K lines | 5 | 78.10 | from scratch |
21
  | S2 | 1.5M lines | 3 | 41.62 | continued |
22
+ | S3 | 87M words | 2 | 28.46 | bf16, cosine LR |
23
  | S4 | 87M words | 3 | 35.16 | grouped context, MLM=0.20 |
24
+ | S5 | 87M words | 2 | 29.67 | fine polish, MLM=0.15 |
25
+ | S6 | 149M words | 2 | 32.76 | ultra fine, LR=3e-6, group=100 |
26
 
27
  ## Usage
28
 
checkpoint-2890/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "RobertaForMaskedLM"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": 1,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": 2,
11
+ "hidden_act": "gelu",
12
+ "hidden_dropout_prob": 0.1,
13
+ "hidden_size": 768,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 3072,
16
+ "is_decoder": false,
17
+ "layer_norm_eps": 1e-12,
18
+ "max_position_embeddings": 514,
19
+ "model_type": "roberta",
20
+ "num_attention_heads": 12,
21
+ "num_hidden_layers": 12,
22
+ "pad_token_id": 0,
23
+ "tie_word_embeddings": true,
24
+ "transformers_version": "5.0.0",
25
+ "type_vocab_size": 1,
26
+ "use_cache": false,
27
+ "vocab_size": 32001
28
+ }
checkpoint-2890/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de2445441ff3ddaabdf20d8cbc0b2ba03dd2dfcf4fc34eed4e79840ea8ee92a5
3
+ size 442633860
checkpoint-2890/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b6af352e86f12dd4dd4fcd1c577c2dfa73c07033b71d5c6432a3f9899104a9f
3
+ size 885391563
checkpoint-2890/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e1b765655777f28541db816119033fbaa94a4a08023c35a19112a494c5d5257
3
+ size 14645
checkpoint-2890/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0dcc058caee8ba6fbd420d0886dccfe545f31ed05f1ab494e66ca392db128922
3
+ size 1465
checkpoint-2890/trainer_state.json ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 2890,
3
+ "best_metric": 3.484435558319092,
4
+ "best_model_checkpoint": "sindhibert_session6/checkpoint-2890",
5
+ "epoch": 1.0,
6
+ "eval_steps": 2890,
7
+ "global_step": 2890,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.03460207612456748,
14
+ "grad_norm": 5.271186828613281,
15
+ "learning_rate": 8.583815028901734e-07,
16
+ "loss": 14.7112890625,
17
+ "step": 100
18
+ },
19
+ {
20
+ "epoch": 0.06920415224913495,
21
+ "grad_norm": 5.328507900238037,
22
+ "learning_rate": 1.7254335260115608e-06,
23
+ "loss": 14.65601806640625,
24
+ "step": 200
25
+ },
26
+ {
27
+ "epoch": 0.10380622837370242,
28
+ "grad_norm": 5.2988433837890625,
29
+ "learning_rate": 2.592485549132948e-06,
30
+ "loss": 14.6437841796875,
31
+ "step": 300
32
+ },
33
+ {
34
+ "epoch": 0.1384083044982699,
35
+ "grad_norm": 5.144472122192383,
36
+ "learning_rate": 2.9992958916410005e-06,
37
+ "loss": 14.644112548828126,
38
+ "step": 400
39
+ },
40
+ {
41
+ "epoch": 0.17301038062283736,
42
+ "grad_norm": 5.255625247955322,
43
+ "learning_rate": 2.994135629312096e-06,
44
+ "loss": 14.592845458984375,
45
+ "step": 500
46
+ },
47
+ {
48
+ "epoch": 0.20761245674740483,
49
+ "grad_norm": 5.365286827087402,
50
+ "learning_rate": 2.9839827352831524e-06,
51
+ "loss": 14.59123779296875,
52
+ "step": 600
53
+ },
54
+ {
55
+ "epoch": 0.2422145328719723,
56
+ "grad_norm": 5.114892959594727,
57
+ "learning_rate": 2.96887113529664e-06,
58
+ "loss": 14.588660888671875,
59
+ "step": 700
60
+ },
61
+ {
62
+ "epoch": 0.2768166089965398,
63
+ "grad_norm": 5.162683486938477,
64
+ "learning_rate": 2.948851324536296e-06,
65
+ "loss": 14.5647998046875,
66
+ "step": 800
67
+ },
68
+ {
69
+ "epoch": 0.31141868512110726,
70
+ "grad_norm": 5.2245635986328125,
71
+ "learning_rate": 2.9239901988982294e-06,
72
+ "loss": 14.566002197265625,
73
+ "step": 900
74
+ },
75
+ {
76
+ "epoch": 0.3460207612456747,
77
+ "grad_norm": 5.22702169418335,
78
+ "learning_rate": 2.8943708314592917e-06,
79
+ "loss": 14.56361328125,
80
+ "step": 1000
81
+ },
82
+ {
83
+ "epoch": 0.3806228373702422,
84
+ "grad_norm": 5.353369235992432,
85
+ "learning_rate": 2.8600921948896393e-06,
86
+ "loss": 14.56337890625,
87
+ "step": 1100
88
+ },
89
+ {
90
+ "epoch": 0.41522491349480967,
91
+ "grad_norm": 5.189014911651611,
92
+ "learning_rate": 2.821268830737051e-06,
93
+ "loss": 14.566427001953125,
94
+ "step": 1200
95
+ },
96
+ {
97
+ "epoch": 0.44982698961937717,
98
+ "grad_norm": 5.108746528625488,
99
+ "learning_rate": 2.7780304666880683e-06,
100
+ "loss": 14.5394287109375,
101
+ "step": 1300
102
+ },
103
+ {
104
+ "epoch": 0.4844290657439446,
105
+ "grad_norm": 5.1513895988464355,
106
+ "learning_rate": 2.7305215830848867e-06,
107
+ "loss": 14.531617431640624,
108
+ "step": 1400
109
+ },
110
+ {
111
+ "epoch": 0.5190311418685121,
112
+ "grad_norm": 5.2011942863464355,
113
+ "learning_rate": 2.678900930146467e-06,
114
+ "loss": 14.5282666015625,
115
+ "step": 1500
116
+ },
117
+ {
118
+ "epoch": 0.5536332179930796,
119
+ "grad_norm": 5.219590663909912,
120
+ "learning_rate": 2.6233409975070707e-06,
121
+ "loss": 14.489625244140624,
122
+ "step": 1600
123
+ },
124
+ {
125
+ "epoch": 0.5882352941176471,
126
+ "grad_norm": 5.411579132080078,
127
+ "learning_rate": 2.5640274378447445e-06,
128
+ "loss": 14.49999755859375,
129
+ "step": 1700
130
+ },
131
+ {
132
+ "epoch": 0.6228373702422145,
133
+ "grad_norm": 5.120180606842041,
134
+ "learning_rate": 2.5011584465256946e-06,
135
+ "loss": 14.515745849609376,
136
+ "step": 1800
137
+ },
138
+ {
139
+ "epoch": 0.657439446366782,
140
+ "grad_norm": 5.3347883224487305,
141
+ "learning_rate": 2.434944099337454e-06,
142
+ "loss": 14.52021728515625,
143
+ "step": 1900
144
+ },
145
+ {
146
+ "epoch": 0.6920415224913494,
147
+ "grad_norm": 4.978041172027588,
148
+ "learning_rate": 2.365605650523803e-06,
149
+ "loss": 14.50703857421875,
150
+ "step": 2000
151
+ },
152
+ {
153
+ "epoch": 0.726643598615917,
154
+ "grad_norm": 5.3864898681640625,
155
+ "learning_rate": 2.293374793467048e-06,
156
+ "loss": 14.493427734375,
157
+ "step": 2100
158
+ },
159
+ {
160
+ "epoch": 0.7612456747404844,
161
+ "grad_norm": 5.149505138397217,
162
+ "learning_rate": 2.2184928864880712e-06,
163
+ "loss": 14.453104248046875,
164
+ "step": 2200
165
+ },
166
+ {
167
+ "epoch": 0.7958477508650519,
168
+ "grad_norm": 5.451559066772461,
169
+ "learning_rate": 2.1412101463511406e-06,
170
+ "loss": 14.48880126953125,
171
+ "step": 2300
172
+ },
173
+ {
174
+ "epoch": 0.8304498269896193,
175
+ "grad_norm": 5.231166839599609,
176
+ "learning_rate": 2.0617848121683582e-06,
177
+ "loss": 14.46824951171875,
178
+ "step": 2400
179
+ },
180
+ {
181
+ "epoch": 0.8650519031141869,
182
+ "grad_norm": 5.132116794586182,
183
+ "learning_rate": 1.9804822824975567e-06,
184
+ "loss": 14.45693603515625,
185
+ "step": 2500
186
+ },
187
+ {
188
+ "epoch": 0.8996539792387543,
189
+ "grad_norm": 5.454078197479248,
190
+ "learning_rate": 1.8975742285170185e-06,
191
+ "loss": 14.468741455078124,
192
+ "step": 2600
193
+ },
194
+ {
195
+ "epoch": 0.9342560553633218,
196
+ "grad_norm": 5.184955596923828,
197
+ "learning_rate": 1.8133376862403233e-06,
198
+ "loss": 14.482393798828125,
199
+ "step": 2700
200
+ },
201
+ {
202
+ "epoch": 0.9688581314878892,
203
+ "grad_norm": 5.030216693878174,
204
+ "learning_rate": 1.7280541308046812e-06,
205
+ "loss": 14.4536962890625,
206
+ "step": 2800
207
+ },
208
+ {
209
+ "epoch": 1.0,
210
+ "eval_loss": 3.484435558319092,
211
+ "eval_runtime": 11.6834,
212
+ "eval_samples_per_second": 639.538,
213
+ "eval_steps_per_second": 10.014,
214
+ "step": 2890
215
+ }
216
+ ],
217
+ "logging_steps": 100,
218
+ "max_steps": 5780,
219
+ "num_input_tokens_seen": 0,
220
+ "num_train_epochs": 2,
221
+ "save_steps": 2890,
222
+ "stateful_callbacks": {
223
+ "TrainerControl": {
224
+ "args": {
225
+ "should_epoch_stop": false,
226
+ "should_evaluate": false,
227
+ "should_log": false,
228
+ "should_save": true,
229
+ "should_training_stop": false
230
+ },
231
+ "attributes": {}
232
+ }
233
+ },
234
+ "total_flos": 1.9471728080409293e+17,
235
+ "train_batch_size": 64,
236
+ "trial_name": null,
237
+ "trial_params": null
238
+ }
checkpoint-2890/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7765c9e92276c99a6c5186a42ac8dca5be5882797e153f8ee76ffcc8912c2d0b
3
+ size 5137
checkpoint-5780/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "RobertaForMaskedLM"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": 1,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": 2,
11
+ "hidden_act": "gelu",
12
+ "hidden_dropout_prob": 0.1,
13
+ "hidden_size": 768,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 3072,
16
+ "is_decoder": false,
17
+ "layer_norm_eps": 1e-12,
18
+ "max_position_embeddings": 514,
19
+ "model_type": "roberta",
20
+ "num_attention_heads": 12,
21
+ "num_hidden_layers": 12,
22
+ "pad_token_id": 0,
23
+ "tie_word_embeddings": true,
24
+ "transformers_version": "5.0.0",
25
+ "type_vocab_size": 1,
26
+ "use_cache": false,
27
+ "vocab_size": 32001
28
+ }
checkpoint-5780/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68752d540c779d0abd4f2ec922d2cee00c47a2a66d63fbd1517f3f2ba6b90f73
3
+ size 442633860
checkpoint-5780/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d6d744c48a0948a89bf0199d108942bd27ab979c14ba8f37f2435071e8129b6
3
+ size 885391563
checkpoint-5780/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68785a4e989d522f15cdea25e8d3966aa30a90e5f9beb086b3fd77f823a1313a
3
+ size 14645
checkpoint-5780/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:410829f8ab4abc91f3407bb46e68effe629fafc369ec90f98afc512104abf985
3
+ size 1465
checkpoint-5780/trainer_state.json ADDED
@@ -0,0 +1,449 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 5780,
3
+ "best_metric": 3.4774351119995117,
4
+ "best_model_checkpoint": "sindhibert_session6/checkpoint-5780",
5
+ "epoch": 2.0,
6
+ "eval_steps": 2890,
7
+ "global_step": 5780,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.03460207612456748,
14
+ "grad_norm": 5.271186828613281,
15
+ "learning_rate": 8.583815028901734e-07,
16
+ "loss": 14.7112890625,
17
+ "step": 100
18
+ },
19
+ {
20
+ "epoch": 0.06920415224913495,
21
+ "grad_norm": 5.328507900238037,
22
+ "learning_rate": 1.7254335260115608e-06,
23
+ "loss": 14.65601806640625,
24
+ "step": 200
25
+ },
26
+ {
27
+ "epoch": 0.10380622837370242,
28
+ "grad_norm": 5.2988433837890625,
29
+ "learning_rate": 2.592485549132948e-06,
30
+ "loss": 14.6437841796875,
31
+ "step": 300
32
+ },
33
+ {
34
+ "epoch": 0.1384083044982699,
35
+ "grad_norm": 5.144472122192383,
36
+ "learning_rate": 2.9992958916410005e-06,
37
+ "loss": 14.644112548828126,
38
+ "step": 400
39
+ },
40
+ {
41
+ "epoch": 0.17301038062283736,
42
+ "grad_norm": 5.255625247955322,
43
+ "learning_rate": 2.994135629312096e-06,
44
+ "loss": 14.592845458984375,
45
+ "step": 500
46
+ },
47
+ {
48
+ "epoch": 0.20761245674740483,
49
+ "grad_norm": 5.365286827087402,
50
+ "learning_rate": 2.9839827352831524e-06,
51
+ "loss": 14.59123779296875,
52
+ "step": 600
53
+ },
54
+ {
55
+ "epoch": 0.2422145328719723,
56
+ "grad_norm": 5.114892959594727,
57
+ "learning_rate": 2.96887113529664e-06,
58
+ "loss": 14.588660888671875,
59
+ "step": 700
60
+ },
61
+ {
62
+ "epoch": 0.2768166089965398,
63
+ "grad_norm": 5.162683486938477,
64
+ "learning_rate": 2.948851324536296e-06,
65
+ "loss": 14.5647998046875,
66
+ "step": 800
67
+ },
68
+ {
69
+ "epoch": 0.31141868512110726,
70
+ "grad_norm": 5.2245635986328125,
71
+ "learning_rate": 2.9239901988982294e-06,
72
+ "loss": 14.566002197265625,
73
+ "step": 900
74
+ },
75
+ {
76
+ "epoch": 0.3460207612456747,
77
+ "grad_norm": 5.22702169418335,
78
+ "learning_rate": 2.8943708314592917e-06,
79
+ "loss": 14.56361328125,
80
+ "step": 1000
81
+ },
82
+ {
83
+ "epoch": 0.3806228373702422,
84
+ "grad_norm": 5.353369235992432,
85
+ "learning_rate": 2.8600921948896393e-06,
86
+ "loss": 14.56337890625,
87
+ "step": 1100
88
+ },
89
+ {
90
+ "epoch": 0.41522491349480967,
91
+ "grad_norm": 5.189014911651611,
92
+ "learning_rate": 2.821268830737051e-06,
93
+ "loss": 14.566427001953125,
94
+ "step": 1200
95
+ },
96
+ {
97
+ "epoch": 0.44982698961937717,
98
+ "grad_norm": 5.108746528625488,
99
+ "learning_rate": 2.7780304666880683e-06,
100
+ "loss": 14.5394287109375,
101
+ "step": 1300
102
+ },
103
+ {
104
+ "epoch": 0.4844290657439446,
105
+ "grad_norm": 5.1513895988464355,
106
+ "learning_rate": 2.7305215830848867e-06,
107
+ "loss": 14.531617431640624,
108
+ "step": 1400
109
+ },
110
+ {
111
+ "epoch": 0.5190311418685121,
112
+ "grad_norm": 5.2011942863464355,
113
+ "learning_rate": 2.678900930146467e-06,
114
+ "loss": 14.5282666015625,
115
+ "step": 1500
116
+ },
117
+ {
118
+ "epoch": 0.5536332179930796,
119
+ "grad_norm": 5.219590663909912,
120
+ "learning_rate": 2.6233409975070707e-06,
121
+ "loss": 14.489625244140624,
122
+ "step": 1600
123
+ },
124
+ {
125
+ "epoch": 0.5882352941176471,
126
+ "grad_norm": 5.411579132080078,
127
+ "learning_rate": 2.5640274378447445e-06,
128
+ "loss": 14.49999755859375,
129
+ "step": 1700
130
+ },
131
+ {
132
+ "epoch": 0.6228373702422145,
133
+ "grad_norm": 5.120180606842041,
134
+ "learning_rate": 2.5011584465256946e-06,
135
+ "loss": 14.515745849609376,
136
+ "step": 1800
137
+ },
138
+ {
139
+ "epoch": 0.657439446366782,
140
+ "grad_norm": 5.3347883224487305,
141
+ "learning_rate": 2.434944099337454e-06,
142
+ "loss": 14.52021728515625,
143
+ "step": 1900
144
+ },
145
+ {
146
+ "epoch": 0.6920415224913494,
147
+ "grad_norm": 4.978041172027588,
148
+ "learning_rate": 2.365605650523803e-06,
149
+ "loss": 14.50703857421875,
150
+ "step": 2000
151
+ },
152
+ {
153
+ "epoch": 0.726643598615917,
154
+ "grad_norm": 5.3864898681640625,
155
+ "learning_rate": 2.293374793467048e-06,
156
+ "loss": 14.493427734375,
157
+ "step": 2100
158
+ },
159
+ {
160
+ "epoch": 0.7612456747404844,
161
+ "grad_norm": 5.149505138397217,
162
+ "learning_rate": 2.2184928864880712e-06,
163
+ "loss": 14.453104248046875,
164
+ "step": 2200
165
+ },
166
+ {
167
+ "epoch": 0.7958477508650519,
168
+ "grad_norm": 5.451559066772461,
169
+ "learning_rate": 2.1412101463511406e-06,
170
+ "loss": 14.48880126953125,
171
+ "step": 2300
172
+ },
173
+ {
174
+ "epoch": 0.8304498269896193,
175
+ "grad_norm": 5.231166839599609,
176
+ "learning_rate": 2.0617848121683582e-06,
177
+ "loss": 14.46824951171875,
178
+ "step": 2400
179
+ },
180
+ {
181
+ "epoch": 0.8650519031141869,
182
+ "grad_norm": 5.132116794586182,
183
+ "learning_rate": 1.9804822824975567e-06,
184
+ "loss": 14.45693603515625,
185
+ "step": 2500
186
+ },
187
+ {
188
+ "epoch": 0.8996539792387543,
189
+ "grad_norm": 5.454078197479248,
190
+ "learning_rate": 1.8975742285170185e-06,
191
+ "loss": 14.468741455078124,
192
+ "step": 2600
193
+ },
194
+ {
195
+ "epoch": 0.9342560553633218,
196
+ "grad_norm": 5.184955596923828,
197
+ "learning_rate": 1.8133376862403233e-06,
198
+ "loss": 14.482393798828125,
199
+ "step": 2700
200
+ },
201
+ {
202
+ "epoch": 0.9688581314878892,
203
+ "grad_norm": 5.030216693878174,
204
+ "learning_rate": 1.7280541308046812e-06,
205
+ "loss": 14.4536962890625,
206
+ "step": 2800
207
+ },
208
+ {
209
+ "epoch": 1.0,
210
+ "eval_loss": 3.484435558319092,
211
+ "eval_runtime": 11.6834,
212
+ "eval_samples_per_second": 639.538,
213
+ "eval_steps_per_second": 10.014,
214
+ "step": 2890
215
+ },
216
+ {
217
+ "epoch": 1.0034602076124568,
218
+ "grad_norm": 5.57059383392334,
219
+ "learning_rate": 1.642008535926004e-06,
220
+ "loss": 14.452493896484375,
221
+ "step": 2900
222
+ },
223
+ {
224
+ "epoch": 1.0380622837370241,
225
+ "grad_norm": 5.143775939941406,
226
+ "learning_rate": 1.555488421663523e-06,
227
+ "loss": 14.467264404296875,
228
+ "step": 3000
229
+ },
230
+ {
231
+ "epoch": 1.0726643598615917,
232
+ "grad_norm": 5.192799091339111,
233
+ "learning_rate": 1.4687828936758435e-06,
234
+ "loss": 14.464970703125,
235
+ "step": 3100
236
+ },
237
+ {
238
+ "epoch": 1.1072664359861593,
239
+ "grad_norm": 5.265404224395752,
240
+ "learning_rate": 1.3821816771787413e-06,
241
+ "loss": 14.4964404296875,
242
+ "step": 3200
243
+ },
244
+ {
245
+ "epoch": 1.1418685121107266,
246
+ "grad_norm": 5.28303337097168,
247
+ "learning_rate": 1.295974148832716e-06,
248
+ "loss": 14.460384521484375,
249
+ "step": 3300
250
+ },
251
+ {
252
+ "epoch": 1.1764705882352942,
253
+ "grad_norm": 5.456825256347656,
254
+ "learning_rate": 1.2104483697952423e-06,
255
+ "loss": 14.472330322265625,
256
+ "step": 3400
257
+ },
258
+ {
259
+ "epoch": 1.2110726643598615,
260
+ "grad_norm": 5.192461013793945,
261
+ "learning_rate": 1.1258901231687475e-06,
262
+ "loss": 14.4242919921875,
263
+ "step": 3500
264
+ },
265
+ {
266
+ "epoch": 1.245674740484429,
267
+ "grad_norm": 5.02662992477417,
268
+ "learning_rate": 1.0425819590606824e-06,
269
+ "loss": 14.4378662109375,
270
+ "step": 3600
271
+ },
272
+ {
273
+ "epoch": 1.2802768166089966,
274
+ "grad_norm": 5.156518459320068,
275
+ "learning_rate": 9.608022504465906e-07,
276
+ "loss": 14.4332080078125,
277
+ "step": 3700
278
+ },
279
+ {
280
+ "epoch": 1.314878892733564,
281
+ "grad_norm": 5.137354850769043,
282
+ "learning_rate": 8.808242629910054e-07,
283
+ "loss": 14.42055419921875,
284
+ "step": 3800
285
+ },
286
+ {
287
+ "epoch": 1.3494809688581315,
288
+ "grad_norm": 4.931822299957275,
289
+ "learning_rate": 8.029152419343472e-07,
290
+ "loss": 14.446268310546875,
291
+ "step": 3900
292
+ },
293
+ {
294
+ "epoch": 1.3840830449826989,
295
+ "grad_norm": 5.438608169555664,
296
+ "learning_rate": 7.273355190969783e-07,
297
+ "loss": 14.436180419921875,
298
+ "step": 4000
299
+ },
300
+ {
301
+ "epoch": 1.4186851211072664,
302
+ "grad_norm": 5.530498027801514,
303
+ "learning_rate": 6.54337642984345e-07,
304
+ "loss": 14.427529296875,
305
+ "step": 4100
306
+ },
307
+ {
308
+ "epoch": 1.453287197231834,
309
+ "grad_norm": 5.223482608795166,
310
+ "learning_rate": 5.841655348999535e-07,
311
+ "loss": 14.415745849609374,
312
+ "step": 4200
313
+ },
314
+ {
315
+ "epoch": 1.4878892733564013,
316
+ "grad_norm": 5.245316982269287,
317
+ "learning_rate": 5.170536738860046e-07,
318
+ "loss": 14.42922119140625,
319
+ "step": 4300
320
+ },
321
+ {
322
+ "epoch": 1.5224913494809689,
323
+ "grad_norm": 5.100404739379883,
324
+ "learning_rate": 4.532263132152101e-07,
325
+ "loss": 14.461243896484374,
326
+ "step": 4400
327
+ },
328
+ {
329
+ "epoch": 1.5570934256055362,
330
+ "grad_norm": 5.252062797546387,
331
+ "learning_rate": 3.928967310518557e-07,
332
+ "loss": 14.419617919921874,
333
+ "step": 4500
334
+ },
335
+ {
336
+ "epoch": 1.5916955017301038,
337
+ "grad_norm": 5.039636611938477,
338
+ "learning_rate": 3.362665177860309e-07,
339
+ "loss": 14.477161865234375,
340
+ "step": 4600
341
+ },
342
+ {
343
+ "epoch": 1.6262975778546713,
344
+ "grad_norm": 5.289979457855225,
345
+ "learning_rate": 2.8352490242237476e-07,
346
+ "loss": 14.38912109375,
347
+ "step": 4700
348
+ },
349
+ {
350
+ "epoch": 1.6608996539792389,
351
+ "grad_norm": 5.163676738739014,
352
+ "learning_rate": 2.348481202742086e-07,
353
+ "loss": 14.403753662109375,
354
+ "step": 4800
355
+ },
356
+ {
357
+ "epoch": 1.6955017301038062,
358
+ "grad_norm": 5.600104331970215,
359
+ "learning_rate": 1.9039882407588976e-07,
360
+ "loss": 14.447159423828126,
361
+ "step": 4900
362
+ },
363
+ {
364
+ "epoch": 1.7301038062283736,
365
+ "grad_norm": 5.240525722503662,
366
+ "learning_rate": 1.503255404811511e-07,
367
+ "loss": 14.401494140625,
368
+ "step": 5000
369
+ },
370
+ {
371
+ "epoch": 1.7647058823529411,
372
+ "grad_norm": 5.186587810516357,
373
+ "learning_rate": 1.1476217376352293e-07,
374
+ "loss": 14.4153125,
375
+ "step": 5100
376
+ },
377
+ {
378
+ "epoch": 1.7993079584775087,
379
+ "grad_norm": 5.226868152618408,
380
+ "learning_rate": 8.382755837722234e-08,
381
+ "loss": 14.4057421875,
382
+ "step": 5200
383
+ },
384
+ {
385
+ "epoch": 1.8339100346020762,
386
+ "grad_norm": 5.335669994354248,
387
+ "learning_rate": 5.762506187361205e-08,
388
+ "loss": 14.435169677734375,
389
+ "step": 5300
390
+ },
391
+ {
392
+ "epoch": 1.8685121107266436,
393
+ "grad_norm": 5.08687686920166,
394
+ "learning_rate": 3.624223950008548e-08,
395
+ "loss": 14.43446533203125,
396
+ "step": 5400
397
+ },
398
+ {
399
+ "epoch": 1.903114186851211,
400
+ "grad_norm": 5.379976272583008,
401
+ "learning_rate": 1.975054163552692e-08,
402
+ "loss": 14.415914306640625,
403
+ "step": 5500
404
+ },
405
+ {
406
+ "epoch": 1.9377162629757785,
407
+ "grad_norm": 5.047989845275879,
408
+ "learning_rate": 8.205075039945553e-09,
409
+ "loss": 14.41682861328125,
410
+ "step": 5600
411
+ },
412
+ {
413
+ "epoch": 1.972318339100346,
414
+ "grad_norm": 5.379687786102295,
415
+ "learning_rate": 1.6444187160679635e-09,
416
+ "loss": 14.4242138671875,
417
+ "step": 5700
418
+ },
419
+ {
420
+ "epoch": 2.0,
421
+ "eval_loss": 3.4774351119995117,
422
+ "eval_runtime": 11.6654,
423
+ "eval_samples_per_second": 640.524,
424
+ "eval_steps_per_second": 10.03,
425
+ "step": 5780
426
+ }
427
+ ],
428
+ "logging_steps": 100,
429
+ "max_steps": 5780,
430
+ "num_input_tokens_seen": 0,
431
+ "num_train_epochs": 2,
432
+ "save_steps": 2890,
433
+ "stateful_callbacks": {
434
+ "TrainerControl": {
435
+ "args": {
436
+ "should_epoch_stop": false,
437
+ "should_evaluate": false,
438
+ "should_log": false,
439
+ "should_save": true,
440
+ "should_training_stop": true
441
+ },
442
+ "attributes": {}
443
+ }
444
+ },
445
+ "total_flos": 3.8943456160818586e+17,
446
+ "train_batch_size": 64,
447
+ "trial_name": null,
448
+ "trial_params": null
449
+ }
checkpoint-5780/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7765c9e92276c99a6c5186a42ac8dca5be5882797e153f8ee76ffcc8912c2d0b
3
+ size 5137
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9f0ec985a6dec33fba0018d1ffbc1fcc4317a2291a75a8d6fc24f269454e6581
3
  size 442633860
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68752d540c779d0abd4f2ec922d2cee00c47a2a66d63fbd1517f3f2ba6b90f73
3
  size 442633860
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a0a0f3ce1e27fd1d47d30a38c39aa86503e7f6fb63f533cef4d2e9ac2db41014
3
  size 5137
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7765c9e92276c99a6c5186a42ac8dca5be5882797e153f8ee76ffcc8912c2d0b
3
  size 5137