Hugging Face's logo

mor1995
/
kanitts2-hebrew-v5b-step3200

Safetensors
lfm2
Model card Files
xet
 Community
mor1995 commited on
Commit
b8b18d2
·
verified ·
1 Parent(s): 30cf9fe

Upload folder using huggingface_hub

Browse files
Files changed (18) hide show
  1. chat_template.jinja +37 -0
  2. config.json +79 -0
  3. model.safetensors +3 -0
  4. optimizer.bin +3 -0
  5. pytorch_model_fsdp.bin +3 -0
  6. rng_state_0.pth +3 -0
  7. rng_state_1.pth +3 -0
  8. rng_state_2.pth +3 -0
  9. rng_state_3.pth +3 -0
  10. rng_state_4.pth +3 -0
  11. rng_state_5.pth +3 -0
  12. rng_state_6.pth +3 -0
  13. rng_state_7.pth +3 -0
  14. scheduler.pt +3 -0
  15. tokenizer.json +0 -0
  16. tokenizer_config.json +19 -0
  17. trainer_state.json +258 -0
  18. training_args.bin +3 -0
chat_template.jinja ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set system_prompt = "" -%}
3
+ {%- set ns = namespace(system_prompt="") -%}
4
+ {%- if messages[0]["role"] == "system" -%}
5
+ {%- set ns.system_prompt = messages[0]["content"] -%}
6
+ {%- set messages = messages[1:] -%}
7
+ {%- endif -%}
8
+ {%- if tools -%}
9
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
10
+ {%- for tool in tools -%}
11
+ {%- if tool is not string -%}
12
+ {%- set tool = tool | tojson -%}
13
+ {%- endif -%}
14
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
15
+ {%- if not loop.last -%}
16
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
17
+ {%- endif -%}
18
+ {%- endfor -%}
19
+ {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
20
+ {%- endif -%}
21
+ {%- if ns.system_prompt -%}
22
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
23
+ {%- endif -%}
24
+ {%- for message in messages -%}
25
+ {{- "<|im_start|>" + message["role"] + "\n" -}}
26
+ {%- set content = message["content"] -%}
27
+ {%- if content is not string -%}
28
+ {%- set content = content | tojson -%}
29
+ {%- endif -%}
30
+ {%- if message["role"] == "tool" -%}
31
+ {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
32
+ {%- endif -%}
33
+ {{- content + "<|im_end|>\n" -}}
34
+ {%- endfor -%}
35
+ {%- if add_generation_prompt -%}
36
+ {{- "<|im_start|>assistant\n" -}}
37
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_max": 2.0,
3
+ "alpha_min": 0.1,
4
+ "architectures": [
5
+ "FlashCompatibleLfm2ForCausalLM"
6
+ ],
7
+ "audio_step": 1.0,
8
+ "block_auto_adjust_ff_dim": true,
9
+ "block_dim": 1024,
10
+ "block_ff_dim": 6656,
11
+ "block_ffn_dim_multiplier": 1.0,
12
+ "block_mlp_init_scale": 1.0,
13
+ "block_multiple_of": 256,
14
+ "block_norm_eps": 1e-05,
15
+ "block_out_init_scale": 1.0,
16
+ "block_use_swiglu": true,
17
+ "block_use_xavier_init": true,
18
+ "bos_token_id": 1,
19
+ "conv_L_cache": 3,
20
+ "conv_bias": false,
21
+ "conv_dim": 1024,
22
+ "conv_dim_out": 1024,
23
+ "conv_use_xavier_init": true,
24
+ "dtype": "float32",
25
+ "eos_token_id": 7,
26
+ "hidden_size": 1024,
27
+ "initializer_range": 0.02,
28
+ "intermediate_size": 6656,
29
+ "language_settings": {
30
+ "language_tags_list": [
31
+ "en_us",
32
+ "en_nyork",
33
+ "en_oakl",
34
+ "en_glasg",
35
+ "en_bost",
36
+ "en_scou"
37
+ ],
38
+ "status": "available_language_tags"
39
+ },
40
+ "layer_types": [
41
+ "conv",
42
+ "conv",
43
+ "full_attention",
44
+ "conv",
45
+ "conv",
46
+ "full_attention",
47
+ "conv",
48
+ "conv",
49
+ "full_attention",
50
+ "conv",
51
+ "full_attention",
52
+ "conv",
53
+ "full_attention",
54
+ "conv",
55
+ "full_attention",
56
+ "conv"
57
+ ],
58
+ "max_position_embeddings": 128000,
59
+ "model_type": "lfm2",
60
+ "norm_eps": 1e-05,
61
+ "num_attention_heads": 16,
62
+ "num_heads": 16,
63
+ "num_hidden_layers": 16,
64
+ "num_key_value_heads": 8,
65
+ "pad_token_id": 0,
66
+ "rope_parameters": {
67
+ "rope_theta": 1000000.0,
68
+ "rope_type": "default"
69
+ },
70
+ "speaker_emb_dim": 128,
71
+ "text_vocab_size": 64400,
72
+ "tie_word_embeddings": true,
73
+ "tokens_per_frame": 4,
74
+ "transformers_version": "5.3.0.dev0",
75
+ "use_cache": false,
76
+ "use_learnable_rope": true,
77
+ "use_pos_enc": true,
78
+ "vocab_size": 80538
79
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb7a688afb88581c3c7f0257bc0fc6d32efcd476bf3b3352e42b42df96ffea80
3
+ size 1809808800
optimizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d3c7a77bf42c813073fbe2c8f771ff40dc7a30032e69c448fe3a2af879aff5d
3
+ size 2959954059
pytorch_model_fsdp.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff133e3267d92314369619b4fbaf62346101484e92b92a5dcf478670c337dda9
3
+ size 1809864263
rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30e15a4fdb02ed1b686cbc96214f76d89cf9fe0f3c5cb2aa94c7c7e2969b6de7
3
+ size 16389
rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa7344a225678a9d0e5eee0d53b85ae9436960d1854bc4a69c6f34dc152cc3e1
3
+ size 16389
rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6af1412df6a4e5329ad5b8d4d0789180caac4593aa5d7b7449cd84f5bfbefa3b
3
+ size 16389
rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69a282d30a0553943e0e9d84f382af59b4c2bf77a11656ad43fb9acd6d5f4039
3
+ size 16389
rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b8b3445137fb7855f3b36c2d2c3d142cab0d64f9c4c8df6a7f430807458166e
3
+ size 16389
rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ec18859fc2d5c79a1a35780890aacdc06758de62c5d6b9041ee5a206961139e
3
+ size 16389
rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b56e05f8f06d30175658631c4b9591e182834c51d939484ca246627597a462f9
3
+ size 16389
rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:273f4447177519084bdf2cb30264b53cebe05a97d2df85a815f948324716b6a0
3
+ size 16389
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2591e85381fedacc66e2172b0e10e341eae0176d01fbbc29f248f1a2d6da8532
3
+ size 1465
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "eos_token": "<|im_end|>",
6
+ "is_local": false,
7
+ "legacy": false,
8
+ "model_input_names": [
9
+ "input_ids",
10
+ "attention_mask"
11
+ ],
12
+ "model_max_length": 1000000000000000019884624838656,
13
+ "pad_token": "<|pad|>",
14
+ "sp_model_kwargs": {},
15
+ "spaces_between_special_tokens": false,
16
+ "tokenizer_class": "TokenizersBackend",
17
+ "use_default_system_prompt": false,
18
+ "use_fast": true
19
+ }
trainer_state.json ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 3.7043150883289893,
6
+ "eval_steps": 500,
7
+ "global_step": 3200,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.11584129742253113,
14
+ "grad_norm": 0.06783366948366165,
15
+ "learning_rate": 1.2375000000000001e-05,
16
+ "loss": 5.668272705078125,
17
+ "step": 100
18
+ },
19
+ {
20
+ "epoch": 0.23168259484506226,
21
+ "grad_norm": 0.09586374461650848,
22
+ "learning_rate": 2.4875e-05,
23
+ "loss": 5.190081176757812,
24
+ "step": 200
25
+ },
26
+ {
27
+ "epoch": 0.3475238922675934,
28
+ "grad_norm": 0.10978876054286957,
29
+ "learning_rate": 3.737500000000001e-05,
30
+ "loss": 4.939649963378907,
31
+ "step": 300
32
+ },
33
+ {
34
+ "epoch": 0.4633651896901245,
35
+ "grad_norm": 0.12643325328826904,
36
+ "learning_rate": 4.9875000000000006e-05,
37
+ "loss": 4.837113037109375,
38
+ "step": 400
39
+ },
40
+ {
41
+ "epoch": 0.5792064871126557,
42
+ "grad_norm": 0.0840177834033966,
43
+ "learning_rate": 4.9971491864977946e-05,
44
+ "loss": 4.77544921875,
45
+ "step": 500
46
+ },
47
+ {
48
+ "epoch": 0.6950477845351868,
49
+ "grad_norm": 0.0854867696762085,
50
+ "learning_rate": 4.988487926942086e-05,
51
+ "loss": 4.728611450195313,
52
+ "step": 600
53
+ },
54
+ {
55
+ "epoch": 0.8108890819577179,
56
+ "grad_norm": 0.08675286173820496,
57
+ "learning_rate": 4.974036084710864e-05,
58
+ "loss": 4.69636962890625,
59
+ "step": 700
60
+ },
61
+ {
62
+ "epoch": 0.926730379380249,
63
+ "grad_norm": 0.07726214826107025,
64
+ "learning_rate": 4.9538272884933975e-05,
65
+ "loss": 4.671690673828125,
66
+ "step": 800
67
+ },
68
+ {
69
+ "epoch": 1.0417028670721111,
70
+ "grad_norm": 0.0845404714345932,
71
+ "learning_rate": 4.927908563113078e-05,
72
+ "loss": 4.64355712890625,
73
+ "step": 900
74
+ },
75
+ {
76
+ "epoch": 1.1575441644946423,
77
+ "grad_norm": 0.07481022924184799,
78
+ "learning_rate": 4.896340220103079e-05,
79
+ "loss": 4.628707275390625,
80
+ "step": 1000
81
+ },
82
+ {
83
+ "epoch": 1.2733854619171736,
84
+ "grad_norm": 0.07507201284170151,
85
+ "learning_rate": 4.8591957173645414e-05,
86
+ "loss": 4.60414306640625,
87
+ "step": 1100
88
+ },
89
+ {
90
+ "epoch": 1.3892267593397047,
91
+ "grad_norm": 0.07688266038894653,
92
+ "learning_rate": 4.8165614882338375e-05,
93
+ "loss": 4.5924462890625,
94
+ "step": 1200
95
+ },
96
+ {
97
+ "epoch": 1.5050680567622359,
98
+ "grad_norm": 0.07709917426109314,
99
+ "learning_rate": 4.768536740356669e-05,
100
+ "loss": 4.576795654296875,
101
+ "step": 1300
102
+ },
103
+ {
104
+ "epoch": 1.620909354184767,
105
+ "grad_norm": 0.07558935880661011,
106
+ "learning_rate": 4.715233224837007e-05,
107
+ "loss": 4.567276611328125,
108
+ "step": 1400
109
+ },
110
+ {
111
+ "epoch": 1.7367506516072981,
112
+ "grad_norm": 0.0724678784608841,
113
+ "learning_rate": 4.6567749761980525e-05,
114
+ "loss": 4.551439819335937,
115
+ "step": 1500
116
+ },
117
+ {
118
+ "epoch": 1.8525919490298293,
119
+ "grad_norm": 0.07796283811330795,
120
+ "learning_rate": 4.59329802376032e-05,
121
+ "loss": 4.538587646484375,
122
+ "step": 1600
123
+ },
124
+ {
125
+ "epoch": 1.9684332464523604,
126
+ "grad_norm": 0.06930229812860489,
127
+ "learning_rate": 4.524950075108444e-05,
128
+ "loss": 4.528701477050781,
129
+ "step": 1700
130
+ },
131
+ {
132
+ "epoch": 2.0834057341442223,
133
+ "grad_norm": 0.07000306248664856,
134
+ "learning_rate": 4.451890172383271e-05,
135
+ "loss": 4.509570922851562,
136
+ "step": 1800
137
+ },
138
+ {
139
+ "epoch": 2.1992470315667534,
140
+ "grad_norm": 0.07318150252103806,
141
+ "learning_rate": 4.3742883221990243e-05,
142
+ "loss": 4.500581665039062,
143
+ "step": 1900
144
+ },
145
+ {
146
+ "epoch": 2.3150883289892845,
147
+ "grad_norm": 0.08026067167520523,
148
+ "learning_rate": 4.2923251000467076e-05,
149
+ "loss": 4.493309020996094,
150
+ "step": 2000
151
+ },
152
+ {
153
+ "epoch": 2.4309296264118156,
154
+ "grad_norm": 0.07233331352472305,
155
+ "learning_rate": 4.2061912301042785e-05,
156
+ "loss": 4.4808154296875,
157
+ "step": 2100
158
+ },
159
+ {
160
+ "epoch": 2.5467709238343472,
161
+ "grad_norm": 0.07325959205627441,
162
+ "learning_rate": 4.1160871414313544e-05,
163
+ "loss": 4.474332885742188,
164
+ "step": 2200
165
+ },
166
+ {
167
+ "epoch": 2.662612221256878,
168
+ "grad_norm": 0.07612094283103943,
169
+ "learning_rate": 4.0222225015811555e-05,
170
+ "loss": 4.462460021972657,
171
+ "step": 2300
172
+ },
173
+ {
174
+ "epoch": 2.7784535186794095,
175
+ "grad_norm": 0.07716099172830582,
176
+ "learning_rate": 3.924815728714959e-05,
177
+ "loss": 4.451640625,
178
+ "step": 2400
179
+ },
180
+ {
181
+ "epoch": 2.89429481610194,
182
+ "grad_norm": 0.07392419874668121,
183
+ "learning_rate": 3.8240934833543305e-05,
184
+ "loss": 4.447383422851562,
185
+ "step": 2500
186
+ },
187
+ {
188
+ "epoch": 3.0092673037938025,
189
+ "grad_norm": 0.07317687571048737,
190
+ "learning_rate": 3.7202901409538247e-05,
191
+ "loss": 4.434151611328125,
192
+ "step": 2600
193
+ },
194
+ {
195
+ "epoch": 3.1251086012163336,
196
+ "grad_norm": 0.07599516957998276,
197
+ "learning_rate": 3.613647246521419e-05,
198
+ "loss": 4.420820617675782,
199
+ "step": 2700
200
+ },
201
+ {
202
+ "epoch": 3.2409498986388647,
203
+ "grad_norm": 0.07734899967908859,
204
+ "learning_rate": 3.504412952555776e-05,
205
+ "loss": 4.41359619140625,
206
+ "step": 2800
207
+ },
208
+ {
209
+ "epoch": 3.356791196061396,
210
+ "grad_norm": 0.07358215004205704,
211
+ "learning_rate": 3.3928414416082205e-05,
212
+ "loss": 4.402438659667968,
213
+ "step": 2900
214
+ },
215
+ {
216
+ "epoch": 3.472632493483927,
217
+ "grad_norm": 0.07404941320419312,
218
+ "learning_rate": 3.279192334813089e-05,
219
+ "loss": 4.402565307617188,
220
+ "step": 3000
221
+ },
222
+ {
223
+ "epoch": 3.588473790906458,
224
+ "grad_norm": 0.08029142022132874,
225
+ "learning_rate": 3.16373008776278e-05,
226
+ "loss": 4.393450927734375,
227
+ "step": 3100
228
+ },
229
+ {
230
+ "epoch": 3.7043150883289893,
231
+ "grad_norm": 0.07287624478340149,
232
+ "learning_rate": 3.0467233751332707e-05,
233
+ "loss": 4.383359985351563,
234
+ "step": 3200
235
+ }
236
+ ],
237
+ "logging_steps": 100,
238
+ "max_steps": 6912,
239
+ "num_input_tokens_seen": 0,
240
+ "num_train_epochs": 8,
241
+ "save_steps": 200,
242
+ "stateful_callbacks": {
243
+ "TrainerControl": {
244
+ "args": {
245
+ "should_epoch_stop": false,
246
+ "should_evaluate": false,
247
+ "should_log": false,
248
+ "should_save": true,
249
+ "should_training_stop": false
250
+ },
251
+ "attributes": {}
252
+ }
253
+ },
254
+ "total_flos": 9.318036855901389e+16,
255
+ "train_batch_size": 4,
256
+ "trial_name": null,
257
+ "trial_params": null
258
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d85da5a57cf3352ac85ee43cf4a04d332e365e590a7f878ed658ef1076a9f1b3
3
+ size 5201