Hugging Face's logo

mor1995
/
kanitts2-hebrew-v5b-final

Safetensors
lfm2
Model card Files
xet
 Community
mor1995 commited on
Commit
21681f2
·
verified ·
1 Parent(s): 55bc8d1

Upload folder using huggingface_hub

Browse files
Files changed (18) hide show
  1. chat_template.jinja +37 -0
  2. config.json +79 -0
  3. model.safetensors +3 -0
  4. optimizer.bin +3 -0
  5. pytorch_model_fsdp.bin +3 -0
  6. rng_state_0.pth +3 -0
  7. rng_state_1.pth +3 -0
  8. rng_state_2.pth +3 -0
  9. rng_state_3.pth +3 -0
  10. rng_state_4.pth +3 -0
  11. rng_state_5.pth +3 -0
  12. rng_state_6.pth +3 -0
  13. rng_state_7.pth +3 -0
  14. scheduler.pt +3 -0
  15. tokenizer.json +0 -0
  16. tokenizer_config.json +19 -0
  17. trainer_state.json +517 -0
  18. training_args.bin +3 -0
chat_template.jinja ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set system_prompt = "" -%}
3
+ {%- set ns = namespace(system_prompt="") -%}
4
+ {%- if messages[0]["role"] == "system" -%}
5
+ {%- set ns.system_prompt = messages[0]["content"] -%}
6
+ {%- set messages = messages[1:] -%}
7
+ {%- endif -%}
8
+ {%- if tools -%}
9
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
10
+ {%- for tool in tools -%}
11
+ {%- if tool is not string -%}
12
+ {%- set tool = tool | tojson -%}
13
+ {%- endif -%}
14
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
15
+ {%- if not loop.last -%}
16
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
17
+ {%- endif -%}
18
+ {%- endfor -%}
19
+ {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
20
+ {%- endif -%}
21
+ {%- if ns.system_prompt -%}
22
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
23
+ {%- endif -%}
24
+ {%- for message in messages -%}
25
+ {{- "<|im_start|>" + message["role"] + "\n" -}}
26
+ {%- set content = message["content"] -%}
27
+ {%- if content is not string -%}
28
+ {%- set content = content | tojson -%}
29
+ {%- endif -%}
30
+ {%- if message["role"] == "tool" -%}
31
+ {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
32
+ {%- endif -%}
33
+ {{- content + "<|im_end|>\n" -}}
34
+ {%- endfor -%}
35
+ {%- if add_generation_prompt -%}
36
+ {{- "<|im_start|>assistant\n" -}}
37
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_max": 2.0,
3
+ "alpha_min": 0.1,
4
+ "architectures": [
5
+ "FlashCompatibleLfm2ForCausalLM"
6
+ ],
7
+ "audio_step": 1.0,
8
+ "block_auto_adjust_ff_dim": true,
9
+ "block_dim": 1024,
10
+ "block_ff_dim": 6656,
11
+ "block_ffn_dim_multiplier": 1.0,
12
+ "block_mlp_init_scale": 1.0,
13
+ "block_multiple_of": 256,
14
+ "block_norm_eps": 1e-05,
15
+ "block_out_init_scale": 1.0,
16
+ "block_use_swiglu": true,
17
+ "block_use_xavier_init": true,
18
+ "bos_token_id": 1,
19
+ "conv_L_cache": 3,
20
+ "conv_bias": false,
21
+ "conv_dim": 1024,
22
+ "conv_dim_out": 1024,
23
+ "conv_use_xavier_init": true,
24
+ "dtype": "float32",
25
+ "eos_token_id": 7,
26
+ "hidden_size": 1024,
27
+ "initializer_range": 0.02,
28
+ "intermediate_size": 6656,
29
+ "language_settings": {
30
+ "language_tags_list": [
31
+ "en_us",
32
+ "en_nyork",
33
+ "en_oakl",
34
+ "en_glasg",
35
+ "en_bost",
36
+ "en_scou"
37
+ ],
38
+ "status": "available_language_tags"
39
+ },
40
+ "layer_types": [
41
+ "conv",
42
+ "conv",
43
+ "full_attention",
44
+ "conv",
45
+ "conv",
46
+ "full_attention",
47
+ "conv",
48
+ "conv",
49
+ "full_attention",
50
+ "conv",
51
+ "full_attention",
52
+ "conv",
53
+ "full_attention",
54
+ "conv",
55
+ "full_attention",
56
+ "conv"
57
+ ],
58
+ "max_position_embeddings": 128000,
59
+ "model_type": "lfm2",
60
+ "norm_eps": 1e-05,
61
+ "num_attention_heads": 16,
62
+ "num_heads": 16,
63
+ "num_hidden_layers": 16,
64
+ "num_key_value_heads": 8,
65
+ "pad_token_id": 0,
66
+ "rope_parameters": {
67
+ "rope_theta": 1000000.0,
68
+ "rope_type": "default"
69
+ },
70
+ "speaker_emb_dim": 128,
71
+ "text_vocab_size": 64400,
72
+ "tie_word_embeddings": true,
73
+ "tokens_per_frame": 4,
74
+ "transformers_version": "5.3.0.dev0",
75
+ "use_cache": false,
76
+ "use_learnable_rope": true,
77
+ "use_pos_enc": true,
78
+ "vocab_size": 80538
79
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:541d9d2cf503408f6eaca878c1559b6366cf3febf1fa905579bbaa27e7fab6be
3
+ size 1809808800
optimizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d0c30f487a9a796fab1a296be24cbac54ea14f0d07d4bacac86f72be8e5dd92
3
+ size 2959954059
pytorch_model_fsdp.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af80ada738515921a276b71b69e82c29e04ac46ab2e65212381449e4d759b023
3
+ size 1809864263
rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:706722fc41436bdb24c4016ab83fbbb62526893d41d4cf49a009fdb7bb4abb5e
3
+ size 16389
rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cd3c5bfb3f44d6324570cdf01e259d25d57145e7c5bcf8a356f6f7b56afdf24
3
+ size 16389
rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ada3d67296c7e68b4d424298393788e9698d123c0f49e21ffd505f02a1def057
3
+ size 16389
rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6abe268928ee44107d52539186bb325f6915b9fe7dafd490b17c58d29bd50783
3
+ size 16389
rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdabaa1468bbe8afa177e64e4dd55a0b2c5ac41e68d229b3d790a6cdd8fbf6fb
3
+ size 16389
rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20c4cea43dfd62d3a3364a4603fb39a847fae2cd95a7b7d8f43fb21cef815cde
3
+ size 16389
rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c22599770b4a3567bdcbd40583c1c29ca81866b4f2fb6dbcf94bafecdef10d2
3
+ size 16389
rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12f20736861ad603467227a1c08b4d09fd872a184e0c54c650a3ac14d130dfcd
3
+ size 16389
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ce84b1be6fc1e945079457ea8d485891a823228860ca486e844c51ee47204e5
3
+ size 1465
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "eos_token": "<|im_end|>",
6
+ "is_local": false,
7
+ "legacy": false,
8
+ "model_input_names": [
9
+ "input_ids",
10
+ "attention_mask"
11
+ ],
12
+ "model_max_length": 1000000000000000019884624838656,
13
+ "pad_token": "<|pad|>",
14
+ "sp_model_kwargs": {},
15
+ "spaces_between_special_tokens": false,
16
+ "tokenizer_class": "TokenizersBackend",
17
+ "use_default_system_prompt": false,
18
+ "use_fast": true
19
+ }
trainer_state.json ADDED
@@ -0,0 +1,517 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 8.0,
6
+ "eval_steps": 500,
7
+ "global_step": 6912,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.11584129742253113,
14
+ "grad_norm": 0.06783366948366165,
15
+ "learning_rate": 1.2375000000000001e-05,
16
+ "loss": 5.668272705078125,
17
+ "step": 100
18
+ },
19
+ {
20
+ "epoch": 0.23168259484506226,
21
+ "grad_norm": 0.09586374461650848,
22
+ "learning_rate": 2.4875e-05,
23
+ "loss": 5.190081176757812,
24
+ "step": 200
25
+ },
26
+ {
27
+ "epoch": 0.3475238922675934,
28
+ "grad_norm": 0.10978876054286957,
29
+ "learning_rate": 3.737500000000001e-05,
30
+ "loss": 4.939649963378907,
31
+ "step": 300
32
+ },
33
+ {
34
+ "epoch": 0.4633651896901245,
35
+ "grad_norm": 0.12643325328826904,
36
+ "learning_rate": 4.9875000000000006e-05,
37
+ "loss": 4.837113037109375,
38
+ "step": 400
39
+ },
40
+ {
41
+ "epoch": 0.5792064871126557,
42
+ "grad_norm": 0.0840177834033966,
43
+ "learning_rate": 4.9971491864977946e-05,
44
+ "loss": 4.77544921875,
45
+ "step": 500
46
+ },
47
+ {
48
+ "epoch": 0.6950477845351868,
49
+ "grad_norm": 0.0854867696762085,
50
+ "learning_rate": 4.988487926942086e-05,
51
+ "loss": 4.728611450195313,
52
+ "step": 600
53
+ },
54
+ {
55
+ "epoch": 0.8108890819577179,
56
+ "grad_norm": 0.08675286173820496,
57
+ "learning_rate": 4.974036084710864e-05,
58
+ "loss": 4.69636962890625,
59
+ "step": 700
60
+ },
61
+ {
62
+ "epoch": 0.926730379380249,
63
+ "grad_norm": 0.07726214826107025,
64
+ "learning_rate": 4.9538272884933975e-05,
65
+ "loss": 4.671690673828125,
66
+ "step": 800
67
+ },
68
+ {
69
+ "epoch": 1.0417028670721111,
70
+ "grad_norm": 0.0845404714345932,
71
+ "learning_rate": 4.927908563113078e-05,
72
+ "loss": 4.64355712890625,
73
+ "step": 900
74
+ },
75
+ {
76
+ "epoch": 1.1575441644946423,
77
+ "grad_norm": 0.07481022924184799,
78
+ "learning_rate": 4.896340220103079e-05,
79
+ "loss": 4.628707275390625,
80
+ "step": 1000
81
+ },
82
+ {
83
+ "epoch": 1.2733854619171736,
84
+ "grad_norm": 0.07507201284170151,
85
+ "learning_rate": 4.8591957173645414e-05,
86
+ "loss": 4.60414306640625,
87
+ "step": 1100
88
+ },
89
+ {
90
+ "epoch": 1.3892267593397047,
91
+ "grad_norm": 0.07688266038894653,
92
+ "learning_rate": 4.8165614882338375e-05,
93
+ "loss": 4.5924462890625,
94
+ "step": 1200
95
+ },
96
+ {
97
+ "epoch": 1.5050680567622359,
98
+ "grad_norm": 0.07709917426109314,
99
+ "learning_rate": 4.768536740356669e-05,
100
+ "loss": 4.576795654296875,
101
+ "step": 1300
102
+ },
103
+ {
104
+ "epoch": 1.620909354184767,
105
+ "grad_norm": 0.07558935880661011,
106
+ "learning_rate": 4.715233224837007e-05,
107
+ "loss": 4.567276611328125,
108
+ "step": 1400
109
+ },
110
+ {
111
+ "epoch": 1.7367506516072981,
112
+ "grad_norm": 0.0724678784608841,
113
+ "learning_rate": 4.6567749761980525e-05,
114
+ "loss": 4.551439819335937,
115
+ "step": 1500
116
+ },
117
+ {
118
+ "epoch": 1.8525919490298293,
119
+ "grad_norm": 0.07796283811330795,
120
+ "learning_rate": 4.59329802376032e-05,
121
+ "loss": 4.538587646484375,
122
+ "step": 1600
123
+ },
124
+ {
125
+ "epoch": 1.9684332464523604,
126
+ "grad_norm": 0.06930229812860489,
127
+ "learning_rate": 4.524950075108444e-05,
128
+ "loss": 4.528701477050781,
129
+ "step": 1700
130
+ },
131
+ {
132
+ "epoch": 2.0834057341442223,
133
+ "grad_norm": 0.07000306248664856,
134
+ "learning_rate": 4.451890172383271e-05,
135
+ "loss": 4.509570922851562,
136
+ "step": 1800
137
+ },
138
+ {
139
+ "epoch": 2.1992470315667534,
140
+ "grad_norm": 0.07318150252103806,
141
+ "learning_rate": 4.3742883221990243e-05,
142
+ "loss": 4.500581665039062,
143
+ "step": 1900
144
+ },
145
+ {
146
+ "epoch": 2.3150883289892845,
147
+ "grad_norm": 0.08026067167520523,
148
+ "learning_rate": 4.2923251000467076e-05,
149
+ "loss": 4.493309020996094,
150
+ "step": 2000
151
+ },
152
+ {
153
+ "epoch": 2.4309296264118156,
154
+ "grad_norm": 0.07233331352472305,
155
+ "learning_rate": 4.2061912301042785e-05,
156
+ "loss": 4.4808154296875,
157
+ "step": 2100
158
+ },
159
+ {
160
+ "epoch": 2.5467709238343472,
161
+ "grad_norm": 0.07325959205627441,
162
+ "learning_rate": 4.1160871414313544e-05,
163
+ "loss": 4.474332885742188,
164
+ "step": 2200
165
+ },
166
+ {
167
+ "epoch": 2.662612221256878,
168
+ "grad_norm": 0.07612094283103943,
169
+ "learning_rate": 4.0222225015811555e-05,
170
+ "loss": 4.462460021972657,
171
+ "step": 2300
172
+ },
173
+ {
174
+ "epoch": 2.7784535186794095,
175
+ "grad_norm": 0.07716099172830582,
176
+ "learning_rate": 3.924815728714959e-05,
177
+ "loss": 4.451640625,
178
+ "step": 2400
179
+ },
180
+ {
181
+ "epoch": 2.89429481610194,
182
+ "grad_norm": 0.07392419874668121,
183
+ "learning_rate": 3.8240934833543305e-05,
184
+ "loss": 4.447383422851562,
185
+ "step": 2500
186
+ },
187
+ {
188
+ "epoch": 3.0092673037938025,
189
+ "grad_norm": 0.07317687571048737,
190
+ "learning_rate": 3.7202901409538247e-05,
191
+ "loss": 4.434151611328125,
192
+ "step": 2600
193
+ },
194
+ {
195
+ "epoch": 3.1251086012163336,
196
+ "grad_norm": 0.07599516957998276,
197
+ "learning_rate": 3.613647246521419e-05,
198
+ "loss": 4.420820617675782,
199
+ "step": 2700
200
+ },
201
+ {
202
+ "epoch": 3.2409498986388647,
203
+ "grad_norm": 0.07734899967908859,
204
+ "learning_rate": 3.504412952555776e-05,
205
+ "loss": 4.41359619140625,
206
+ "step": 2800
207
+ },
208
+ {
209
+ "epoch": 3.356791196061396,
210
+ "grad_norm": 0.07358215004205704,
211
+ "learning_rate": 3.3928414416082205e-05,
212
+ "loss": 4.402438659667968,
213
+ "step": 2900
214
+ },
215
+ {
216
+ "epoch": 3.472632493483927,
217
+ "grad_norm": 0.07404941320419312,
218
+ "learning_rate": 3.279192334813089e-05,
219
+ "loss": 4.402565307617188,
220
+ "step": 3000
221
+ },
222
+ {
223
+ "epoch": 3.588473790906458,
224
+ "grad_norm": 0.08029142022132874,
225
+ "learning_rate": 3.16373008776278e-05,
226
+ "loss": 4.393450927734375,
227
+ "step": 3100
228
+ },
229
+ {
230
+ "epoch": 3.7043150883289893,
231
+ "grad_norm": 0.07287624478340149,
232
+ "learning_rate": 3.0467233751332707e-05,
233
+ "loss": 4.383359985351563,
234
+ "step": 3200
235
+ },
236
+ {
237
+ "epoch": 3.8201563857515204,
238
+ "grad_norm": 0.0857250988483429,
239
+ "learning_rate": 2.9284444654920458e-05,
240
+ "loss": 4.37967041015625,
241
+ "step": 3300
242
+ },
243
+ {
244
+ "epoch": 3.9359976831740515,
245
+ "grad_norm": 0.07641367614269257,
246
+ "learning_rate": 2.8091685877432245e-05,
247
+ "loss": 4.374575805664063,
248
+ "step": 3400
249
+ },
250
+ {
251
+ "epoch": 4.050970170865914,
252
+ "grad_norm": 0.0765901580452919,
253
+ "learning_rate": 2.689173290684141e-05,
254
+ "loss": 4.36169677734375,
255
+ "step": 3500
256
+ },
257
+ {
258
+ "epoch": 4.1668114682884445,
259
+ "grad_norm": 0.0748104378581047,
260
+ "learning_rate": 2.5687377971636452e-05,
261
+ "loss": 4.350984191894531,
262
+ "step": 3600
263
+ },
264
+ {
265
+ "epoch": 4.282652765710976,
266
+ "grad_norm": 0.0798560157418251,
267
+ "learning_rate": 2.448142354344983e-05,
268
+ "loss": 4.345054321289062,
269
+ "step": 3700
270
+ },
271
+ {
272
+ "epoch": 4.398494063133507,
273
+ "grad_norm": 0.08187977224588394,
274
+ "learning_rate": 2.327667581585144e-05,
275
+ "loss": 4.336292419433594,
276
+ "step": 3800
277
+ },
278
+ {
279
+ "epoch": 4.514335360556038,
280
+ "grad_norm": 0.07813862711191177,
281
+ "learning_rate": 2.20759381744813e-05,
282
+ "loss": 4.331329345703125,
283
+ "step": 3900
284
+ },
285
+ {
286
+ "epoch": 4.630176657978569,
287
+ "grad_norm": 0.07801263779401779,
288
+ "learning_rate": 2.088200467371626e-05,
289
+ "loss": 4.328626708984375,
290
+ "step": 4000
291
+ },
292
+ {
293
+ "epoch": 4.746017955401101,
294
+ "grad_norm": 0.07859046012163162,
295
+ "learning_rate": 1.9697653535049934e-05,
296
+ "loss": 4.317982177734375,
297
+ "step": 4100
298
+ },
299
+ {
300
+ "epoch": 4.861859252823631,
301
+ "grad_norm": 0.08107456564903259,
302
+ "learning_rate": 1.852564068231503e-05,
303
+ "loss": 4.315095825195312,
304
+ "step": 4200
305
+ },
306
+ {
307
+ "epoch": 4.977700550246163,
308
+ "grad_norm": 0.07920628041028976,
309
+ "learning_rate": 1.736869332879118e-05,
310
+ "loss": 4.309036865234375,
311
+ "step": 4300
312
+ },
313
+ {
314
+ "epoch": 5.092673037938025,
315
+ "grad_norm": 0.08328945934772491,
316
+ "learning_rate": 1.6229503631120664e-05,
317
+ "loss": 4.300845642089843,
318
+ "step": 4400
319
+ },
320
+ {
321
+ "epoch": 5.208514335360556,
322
+ "grad_norm": 0.07854989916086197,
323
+ "learning_rate": 1.5110722424799245e-05,
324
+ "loss": 4.284882507324219,
325
+ "step": 4500
326
+ },
327
+ {
328
+ "epoch": 5.324355632783087,
329
+ "grad_norm": 0.08098334074020386,
330
+ "learning_rate": 1.4014953055819014e-05,
331
+ "loss": 4.286058349609375,
332
+ "step": 4600
333
+ },
334
+ {
335
+ "epoch": 5.440196930205619,
336
+ "grad_norm": 0.08080381900072098,
337
+ "learning_rate": 1.2944745322817014e-05,
338
+ "loss": 4.287008056640625,
339
+ "step": 4700
340
+ },
341
+ {
342
+ "epoch": 5.556038227628149,
343
+ "grad_norm": 0.08096092194318771,
344
+ "learning_rate": 1.1902589543825637e-05,
345
+ "loss": 4.283847045898438,
346
+ "step": 4800
347
+ },
348
+ {
349
+ "epoch": 5.671879525050681,
350
+ "grad_norm": 0.0861465260386467,
351
+ "learning_rate": 1.0890910761431491e-05,
352
+ "loss": 4.276922607421875,
353
+ "step": 4900
354
+ },
355
+ {
356
+ "epoch": 5.7877208224732115,
357
+ "grad_norm": 0.08496175706386566,
358
+ "learning_rate": 9.912063099826655e-06,
359
+ "loss": 4.274569091796875,
360
+ "step": 5000
361
+ },
362
+ {
363
+ "epoch": 5.903562119895743,
364
+ "grad_norm": 0.08237385749816895,
365
+ "learning_rate": 8.968324286883479e-06,
366
+ "loss": 4.269696044921875,
367
+ "step": 5100
368
+ },
369
+ {
370
+ "epoch": 6.018534607587605,
371
+ "grad_norm": 0.08620701730251312,
372
+ "learning_rate": 8.061890353999706e-06,
373
+ "loss": 4.269225463867188,
374
+ "step": 5200
375
+ },
376
+ {
377
+ "epoch": 6.1343759050101365,
378
+ "grad_norm": 0.0817682072520256,
379
+ "learning_rate": 7.194870526046971e-06,
380
+ "loss": 4.258948974609375,
381
+ "step": 5300
382
+ },
383
+ {
384
+ "epoch": 6.250217202432667,
385
+ "grad_norm": 0.08689381927251816,
386
+ "learning_rate": 6.3692823133136265e-06,
387
+ "loss": 4.2539605712890625,
388
+ "step": 5400
389
+ },
390
+ {
391
+ "epoch": 6.366058499855199,
392
+ "grad_norm": 0.08615448325872421,
393
+ "learning_rate": 5.5870468168627094e-06,
394
+ "loss": 4.252815856933593,
395
+ "step": 5500
396
+ },
397
+ {
398
+ "epoch": 6.4818997972777295,
399
+ "grad_norm": 0.08736546337604523,
400
+ "learning_rate": 4.849984258229085e-06,
401
+ "loss": 4.248274536132812,
402
+ "step": 5600
403
+ },
404
+ {
405
+ "epoch": 6.597741094700261,
406
+ "grad_norm": 0.08700817078351974,
407
+ "learning_rate": 4.159809743858159e-06,
408
+ "loss": 4.251017761230469,
409
+ "step": 5700
410
+ },
411
+ {
412
+ "epoch": 6.713582392122792,
413
+ "grad_norm": 0.08986040949821472,
414
+ "learning_rate": 3.5181292741418707e-06,
415
+ "loss": 4.244992065429687,
416
+ "step": 5800
417
+ },
418
+ {
419
+ "epoch": 6.829423689545322,
420
+ "grad_norm": 0.08741279691457748,
421
+ "learning_rate": 2.926436006338862e-06,
422
+ "loss": 4.251634216308593,
423
+ "step": 5900
424
+ },
425
+ {
426
+ "epoch": 6.945264986967854,
427
+ "grad_norm": 0.09081193059682846,
428
+ "learning_rate": 2.3861067800747845e-06,
429
+ "loss": 4.247467651367187,
430
+ "step": 6000
431
+ },
432
+ {
433
+ "epoch": 7.060237474659716,
434
+ "grad_norm": 0.085789255797863,
435
+ "learning_rate": 1.8983989135076902e-06,
436
+ "loss": 4.245743713378906,
437
+ "step": 6100
438
+ },
439
+ {
440
+ "epoch": 7.176078772082247,
441
+ "grad_norm": 0.08582032471895218,
442
+ "learning_rate": 1.464447277613773e-06,
443
+ "loss": 4.243508911132812,
444
+ "step": 6200
445
+ },
446
+ {
447
+ "epoch": 7.291920069504778,
448
+ "grad_norm": 0.08856623619794846,
449
+ "learning_rate": 1.0852616554012623e-06,
450
+ "loss": 4.240311279296875,
451
+ "step": 6300
452
+ },
453
+ {
454
+ "epoch": 7.40776136692731,
455
+ "grad_norm": 0.08958588540554047,
456
+ "learning_rate": 7.617243921976214e-07,
457
+ "loss": 4.236758728027343,
458
+ "step": 6400
459
+ },
460
+ {
461
+ "epoch": 7.52360266434984,
462
+ "grad_norm": 0.08384504914283752,
463
+ "learning_rate": 4.945883424776088e-07,
464
+ "loss": 4.239651489257812,
465
+ "step": 6500
466
+ },
467
+ {
468
+ "epoch": 7.639443961772372,
469
+ "grad_norm": 0.08597233146429062,
470
+ "learning_rate": 2.8447511800987205e-07,
471
+ "loss": 4.236366577148438,
472
+ "step": 6600
473
+ },
474
+ {
475
+ "epoch": 7.755285259194903,
476
+ "grad_norm": 0.089970164000988,
477
+ "learning_rate": 1.3187364139850555e-07,
478
+ "loss": 4.234116821289063,
479
+ "step": 6700
480
+ },
481
+ {
482
+ "epoch": 7.871126556617434,
483
+ "grad_norm": 0.083633191883564,
484
+ "learning_rate": 3.713900838545059e-08,
485
+ "loss": 4.235526733398437,
486
+ "step": 6800
487
+ },
488
+ {
489
+ "epoch": 7.986967854039965,
490
+ "grad_norm": 0.08615066856145859,
491
+ "learning_rate": 4.916615610639496e-10,
492
+ "loss": 4.238587036132812,
493
+ "step": 6900
494
+ }
495
+ ],
496
+ "logging_steps": 100,
497
+ "max_steps": 6912,
498
+ "num_input_tokens_seen": 0,
499
+ "num_train_epochs": 8,
500
+ "save_steps": 200,
501
+ "stateful_callbacks": {
502
+ "TrainerControl": {
503
+ "args": {
504
+ "should_epoch_stop": false,
505
+ "should_evaluate": false,
506
+ "should_log": false,
507
+ "should_save": true,
508
+ "should_training_stop": true
509
+ },
510
+ "attributes": {}
511
+ }
512
+ },
513
+ "total_flos": 2.0119176163518054e+17,
514
+ "train_batch_size": 4,
515
+ "trial_name": null,
516
+ "trial_params": null
517
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d85da5a57cf3352ac85ee43cf4a04d332e365e590a7f878ed658ef1076a9f1b3
3
+ size 5201