|
|
from tokenizers import Regex, Tokenizer, decoders, pre_tokenizers, processors |
|
|
from tokenizers.models import BPE |
|
|
|
|
|
from transformers import LlamaTokenizerFast |
|
|
from transformers.convert_slow_tokenizer import bytes_to_unicode |
|
|
|
|
|
|
|
|
class MistralConverter: |
|
|
""" |
|
|
A general tiktoken converter. |
|
|
""" |
|
|
|
|
|
def __init__( |
|
|
self, |
|
|
vocab=None, |
|
|
pattern=r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""", |
|
|
add_prefix_space=False, |
|
|
additional_special_tokens=None, |
|
|
**kwargs, |
|
|
): |
|
|
self.vocab = vocab |
|
|
self.pattern = pattern |
|
|
self.add_prefix_space = add_prefix_space |
|
|
self.additional_special_tokens = additional_special_tokens |
|
|
|
|
|
def extract_vocab_merges_from_model(self, vocab: str): |
|
|
bpe_ranks = vocab |
|
|
byte_encoder = bytes_to_unicode() |
|
|
|
|
|
def token_bytes_to_string(b): |
|
|
return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")]) |
|
|
|
|
|
merges = [] |
|
|
vocab = {} |
|
|
for idx, (token, rank) in enumerate(bpe_ranks.items()): |
|
|
if token not in self.additional_special_tokens: |
|
|
vocab[token_bytes_to_string(token)] = idx |
|
|
if len(token) == 1: |
|
|
continue |
|
|
local = [] |
|
|
for index in range(1, len(token)): |
|
|
piece_l, piece_r = token[:index], token[index:] |
|
|
if piece_l in bpe_ranks and piece_r in bpe_ranks and (piece_l + piece_r) in bpe_ranks: |
|
|
local.append((piece_l, piece_r, rank)) |
|
|
local = sorted(local, key=lambda x: (bpe_ranks[x[0]], bpe_ranks[x[1]]), reverse=False) |
|
|
merges.extend(local) |
|
|
else: |
|
|
vocab[token] = idx |
|
|
merges = sorted(merges, key=lambda val: val[2], reverse=False) |
|
|
merges = [(token_bytes_to_string(val[0]), token_bytes_to_string(val[1])) for val in merges] |
|
|
return vocab, merges |
|
|
|
|
|
def tokenizer(self): |
|
|
vocab_scores, merges = self.extract_vocab_merges_from_model(self.vocab) |
|
|
tokenizer = Tokenizer(BPE(vocab_scores, merges, fuse_unk=False)) |
|
|
if hasattr(tokenizer.model, "ignore_merges"): |
|
|
tokenizer.model.ignore_merges = True |
|
|
return tokenizer |
|
|
|
|
|
def converted(self) -> Tokenizer: |
|
|
tokenizer = self.tokenizer() |
|
|
tokenizer.pre_tokenizer = pre_tokenizers.Sequence( |
|
|
[ |
|
|
pre_tokenizers.Split(Regex(self.pattern), behavior="isolated", invert=False), |
|
|
pre_tokenizers.ByteLevel(add_prefix_space=self.add_prefix_space, use_regex=False), |
|
|
] |
|
|
) |
|
|
tokenizer.decoder = decoders.ByteLevel() |
|
|
tokenizer.add_special_tokens(self.additional_special_tokens) |
|
|
|
|
|
tokenizer.post_processor = processors.ByteLevel(trim_offsets=False) |
|
|
|
|
|
return tokenizer |
|
|
|
|
|
|
|
|
def convert_tekken_tokenizer(tokenizer_file: str): |
|
|
"""Convert a "tekken" tokenizer to a fast Tokenizer.""" |
|
|
|
|
|
|
|
|
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer |
|
|
|
|
|
|
|
|
mistral_tokenizer = MistralTokenizer.from_file(tokenizer_file) |
|
|
|
|
|
|
|
|
vocab = mistral_tokenizer.instruct_tokenizer.tokenizer._tekken_token2id_nospecial |
|
|
all_special = [ |
|
|
token.value if hasattr(token, "value") else token |
|
|
for token in mistral_tokenizer.instruct_tokenizer.tokenizer._all_special_tokens |
|
|
] |
|
|
specials_tokens = {token: all_special.index(token) for token in all_special} |
|
|
specials_tokens.update(vocab) |
|
|
vocab = specials_tokens |
|
|
|
|
|
|
|
|
tokenizer = LlamaTokenizerFast( |
|
|
tokenizer_object=MistralConverter(vocab=vocab, additional_special_tokens=all_special).converted(), |
|
|
) |
|
|
|
|
|
|
|
|
tokenizer.add_special_tokens({"additional_special_tokens": all_special}) |
|
|
|
|
|
return tokenizer |
|
|
|
