Upload ultravox_tokenizer.py with huggingface_hub

ultravox_tokenizer.py

@@ -0,0 +1,25 @@
+import logging
+
+import transformers
+
+AUDIO_TOKEN = "<|audio|>"
+
+
+def from_pretrained_text_tokenizer(
+    *args, **kwargs
+) -> transformers.PreTrainedTokenizerBase:
+    """
+    Create a tokenizer with the additional special token for audio.
+    This is mainly used for VLLM to work properly. This repo does not currently require it.
+    """
+
+    tokenizer = transformers.AutoTokenizer.from_pretrained(*args, **kwargs)
+    tokenizer.add_special_tokens({"additional_special_tokens": [AUDIO_TOKEN]})
+    logging.info(f"Audio token id: {get_audio_token_id(tokenizer)}")
+    return tokenizer
+
+
+def get_audio_token_id(tokenizer: transformers.PreTrainedTokenizerBase) -> int:
+    audio_token_id = tokenizer.encode(AUDIO_TOKEN, add_special_tokens=False)
+    assert len(audio_token_id) == 1, "Audio token should be a single token"
+    return audio_token_id[0]