diff --git "a/tokenizer_config.json" "b/tokenizer_config.json" new file mode 100644--- /dev/null +++ "b/tokenizer_config.json" @@ -0,0 +1,493 @@ + + + + + + + + + + + + + + + + + + + + + + + + tokenizer_config.json · openai/whisper-medium at main + + + + +
+
Hugging Face's logo + +
+ +
+
+ +
+
+ +
+ + +
+ +

+
+ +
+ openai +
/
+ +
whisper-medium +
+
+
+ +

+
+ Automatic Speech Recognition + + PyTorch + + TensorFlow + + JAX + + Transformers + + + 99 languages + + whisper + + audio + + hf-asr-leaderboard + + Eval Results +
+ + + + +
+ License: + apache-2.0 +
+ +
+ Model card + + + + Files + + + + + Community +
19 +
+ + +
+ + + + +
+ +
+ + + + +
+ + + + + + + +
+ + + + +
+
+
+ + + + +
+
+
+
+
+ +
+ + + + +
+
whisper-medium + / + tokenizer_config.json
+ + +
+
ArthurZ's picture +
ArthurZ + +
HF staff +
+
+
Changes for fast tokenizer (#7)
+ bca8750 + +
+
+ + raw + + history + + blame + + contribute + + delete + +
+ + No virus +
+ +
843 Bytes
+ +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
{
"add_bos_token": false,
"add_prefix_space": false,
"bos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"errors": "replace",
"model_max_length": 1024,
"name_or_path": "openai/whisper-medium",
"pad_token": null,
"processor_class": "WhisperProcessor",
"return_attention_mask": false,
"special_tokens_map_file": null,
"tokenizer_class": "WhisperTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}
+
+
+
+ + + + + + + + + + + + +