Hezar: Upload tokenizer_config.yaml

preprocessor/tokenizer_config.yaml

@@ -1,6 +1,5 @@
 name: whisper_bpe_tokenizer
 config_type: preprocessor
-pretrained_path: hezarai/whisper-small
 max_length: 512
 truncation_strategy: longest_first
 truncation_direction: right
@@ -8,11 +7,15 @@ stride: 0
 padding_strategy: longest
 padding_direction: right
 pad_to_multiple_of: 0
-pad_token_id: 0
-pad_token: <pad>
 pad_token_type_id: 0
+bos_token: <|startoftranscript|>
+eos_token: <|endoftext|>
 unk_token: <|endoftext|>
-special_tokens:
+sep_token: <sep>
+pad_token: <pad>
+cls_token: <cls>
+mask_token: <mask>
+additional_special_tokens:
 - <|endoftext|>
 - <|endoftext|>
 - <|startoftranscript|>
@@ -124,16 +127,14 @@ special_tokens:
 continuing_subword_prefix: ''
 end_of_word_suffix: ''
 fuse_unk: false
-vocab_size: 50364
+vocab_size: 30000
 min_frequency: 2
 limit_alphabet: 1000
 initial_alphabet: []
 show_progress: true
-unk_token_id: 50257
-bos_token: <|startoftranscript|>
-bos_token_id: 50257
-eos_token: <|endoftext|>
-eos_token_id: 50257
+translate_token: <|translate|>
+transcribe_token: <|transcribe|>
+notimestamps_token: <|notimestamps|>
 add_prefix_space: false
 add_bos_token: false
 model_max_length: 1024