Add loan-advisor-agent
Browse files- .gitattributes +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/__init__.py +8 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_educational.py +223 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/core.py +446 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/load.py +172 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/model.py +118 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/py.typed +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/tiktoken/registry.py +96 -0
.gitattributes
CHANGED
|
@@ -708,3 +708,4 @@ loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_random.cp312-win_amd64
|
|
| 708 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_seq_dataset.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 709 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/sparsefuncs_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 710 |
loan-advisor-agent/.venv/Lib/site-packages/sqlalchemy/cyextension/collections.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 708 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_seq_dataset.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 709 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/sparsefuncs_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 710 |
loan-advisor-agent/.venv/Lib/site-packages/sqlalchemy/cyextension/collections.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 711 |
+
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This is the public API of tiktoken
|
| 2 |
+
from .core import Encoding as Encoding
|
| 3 |
+
from .model import encoding_for_model as encoding_for_model
|
| 4 |
+
from .model import encoding_name_for_model as encoding_name_for_model
|
| 5 |
+
from .registry import get_encoding as get_encoding
|
| 6 |
+
from .registry import list_encoding_names as list_encoding_names
|
| 7 |
+
|
| 8 |
+
__version__ = "0.12.0"
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_educational.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This is an educational implementation of the byte pair encoding algorithm."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import collections
|
| 6 |
+
|
| 7 |
+
import regex
|
| 8 |
+
|
| 9 |
+
import tiktoken
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class SimpleBytePairEncoding:
|
| 13 |
+
def __init__(self, *, pat_str: str, mergeable_ranks: dict[bytes, int]) -> None:
|
| 14 |
+
"""Creates an Encoding object."""
|
| 15 |
+
# A regex pattern string that is used to split the input text
|
| 16 |
+
self.pat_str = pat_str
|
| 17 |
+
# A dictionary mapping token bytes to their ranks. The ranks correspond to merge priority
|
| 18 |
+
self.mergeable_ranks = mergeable_ranks
|
| 19 |
+
|
| 20 |
+
self._decoder = {token: token_bytes for token_bytes, token in mergeable_ranks.items()}
|
| 21 |
+
self._pat = regex.compile(pat_str)
|
| 22 |
+
|
| 23 |
+
def encode(self, text: str, visualise: str | None = "colour") -> list[int]:
|
| 24 |
+
"""Encodes a string into tokens.
|
| 25 |
+
|
| 26 |
+
>>> enc.encode("hello world")
|
| 27 |
+
[388, 372]
|
| 28 |
+
"""
|
| 29 |
+
# Use the regex to split the text into (approximately) words
|
| 30 |
+
words = self._pat.findall(text)
|
| 31 |
+
tokens = []
|
| 32 |
+
for word in words:
|
| 33 |
+
# Turn each word into tokens, using the byte pair encoding algorithm
|
| 34 |
+
word_bytes = word.encode("utf-8")
|
| 35 |
+
word_tokens = bpe_encode(self.mergeable_ranks, word_bytes, visualise=visualise)
|
| 36 |
+
tokens.extend(word_tokens)
|
| 37 |
+
return tokens
|
| 38 |
+
|
| 39 |
+
def decode_bytes(self, tokens: list[int]) -> bytes:
|
| 40 |
+
"""Decodes a list of tokens into bytes.
|
| 41 |
+
|
| 42 |
+
>>> enc.decode_bytes([388, 372])
|
| 43 |
+
b'hello world'
|
| 44 |
+
"""
|
| 45 |
+
return b"".join(self._decoder[token] for token in tokens)
|
| 46 |
+
|
| 47 |
+
def decode(self, tokens: list[int]) -> str:
|
| 48 |
+
"""Decodes a list of tokens into a string.
|
| 49 |
+
|
| 50 |
+
Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
|
| 51 |
+
the invalid bytes with the replacement character "�".
|
| 52 |
+
|
| 53 |
+
>>> enc.decode([388, 372])
|
| 54 |
+
'hello world'
|
| 55 |
+
"""
|
| 56 |
+
return self.decode_bytes(tokens).decode("utf-8", errors="replace")
|
| 57 |
+
|
| 58 |
+
def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]:
|
| 59 |
+
"""Decodes a list of tokens into a list of bytes.
|
| 60 |
+
|
| 61 |
+
Useful for visualising how a string is tokenised.
|
| 62 |
+
|
| 63 |
+
>>> enc.decode_tokens_bytes([388, 372])
|
| 64 |
+
[b'hello', b' world']
|
| 65 |
+
"""
|
| 66 |
+
return [self._decoder[token] for token in tokens]
|
| 67 |
+
|
| 68 |
+
@staticmethod
|
| 69 |
+
def train(training_data: str, vocab_size: int, pat_str: str):
|
| 70 |
+
"""Train a BPE tokeniser on some data!"""
|
| 71 |
+
mergeable_ranks = bpe_train(data=training_data, vocab_size=vocab_size, pat_str=pat_str)
|
| 72 |
+
return SimpleBytePairEncoding(pat_str=pat_str, mergeable_ranks=mergeable_ranks)
|
| 73 |
+
|
| 74 |
+
@staticmethod
|
| 75 |
+
def from_tiktoken(encoding):
|
| 76 |
+
if isinstance(encoding, str):
|
| 77 |
+
encoding = tiktoken.get_encoding(encoding)
|
| 78 |
+
return SimpleBytePairEncoding(
|
| 79 |
+
pat_str=encoding._pat_str, mergeable_ranks=encoding._mergeable_ranks
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def bpe_encode(
|
| 84 |
+
mergeable_ranks: dict[bytes, int], input: bytes, visualise: str | None = "colour"
|
| 85 |
+
) -> list[int]:
|
| 86 |
+
parts = [bytes([b]) for b in input]
|
| 87 |
+
while True:
|
| 88 |
+
# See the intermediate merges play out!
|
| 89 |
+
if visualise:
|
| 90 |
+
if visualise in ["colour", "color"]:
|
| 91 |
+
visualise_tokens(parts)
|
| 92 |
+
elif visualise == "simple":
|
| 93 |
+
print(parts)
|
| 94 |
+
|
| 95 |
+
# Iterate over all pairs and find the pair we want to merge the most
|
| 96 |
+
min_idx = None
|
| 97 |
+
min_rank = None
|
| 98 |
+
for i, pair in enumerate(zip(parts[:-1], parts[1:])):
|
| 99 |
+
rank = mergeable_ranks.get(pair[0] + pair[1])
|
| 100 |
+
if rank is not None and (min_rank is None or rank < min_rank):
|
| 101 |
+
min_idx = i
|
| 102 |
+
min_rank = rank
|
| 103 |
+
|
| 104 |
+
# If there were no pairs we could merge, we're done!
|
| 105 |
+
if min_rank is None:
|
| 106 |
+
break
|
| 107 |
+
assert min_idx is not None
|
| 108 |
+
|
| 109 |
+
# Otherwise, merge that pair and leave the rest unchanged. Then repeat.
|
| 110 |
+
parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]
|
| 111 |
+
|
| 112 |
+
if visualise:
|
| 113 |
+
print()
|
| 114 |
+
|
| 115 |
+
tokens = [mergeable_ranks[part] for part in parts]
|
| 116 |
+
return tokens
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def bpe_train(
|
| 120 |
+
data: str, vocab_size: int, pat_str: str, visualise: str | None = "colour"
|
| 121 |
+
) -> dict[bytes, int]:
|
| 122 |
+
# First, add tokens for each individual byte value
|
| 123 |
+
if vocab_size < 2**8:
|
| 124 |
+
raise ValueError("vocab_size must be at least 256, so we can encode all bytes")
|
| 125 |
+
ranks = {}
|
| 126 |
+
for i in range(2**8):
|
| 127 |
+
ranks[bytes([i])] = i
|
| 128 |
+
|
| 129 |
+
# Splinter up our data into lists of bytes
|
| 130 |
+
# data = "Hello world"
|
| 131 |
+
# words = [
|
| 132 |
+
# [b'H', b'e', b'l', b'l', b'o'],
|
| 133 |
+
# [b' ', b'w', b'o', b'r', b'l', b'd']
|
| 134 |
+
# ]
|
| 135 |
+
words: list[list[bytes]] = [
|
| 136 |
+
[bytes([b]) for b in word.encode("utf-8")] for word in regex.findall(pat_str, data)
|
| 137 |
+
]
|
| 138 |
+
|
| 139 |
+
# Now, use our data to figure out which merges we should make
|
| 140 |
+
while len(ranks) < vocab_size:
|
| 141 |
+
# Find the most common pair. This will become our next token
|
| 142 |
+
stats = collections.Counter()
|
| 143 |
+
for piece in words:
|
| 144 |
+
for pair in zip(piece[:-1], piece[1:]):
|
| 145 |
+
stats[pair] += 1
|
| 146 |
+
|
| 147 |
+
most_common_pair = max(stats, key=lambda x: stats[x])
|
| 148 |
+
token_bytes = most_common_pair[0] + most_common_pair[1]
|
| 149 |
+
token = len(ranks)
|
| 150 |
+
# Add the new token!
|
| 151 |
+
ranks[token_bytes] = token
|
| 152 |
+
|
| 153 |
+
# Now merge that most common pair in all the words. That is, update our training data
|
| 154 |
+
# to reflect our decision to make that pair into a new token.
|
| 155 |
+
new_words = []
|
| 156 |
+
for word in words:
|
| 157 |
+
new_word = []
|
| 158 |
+
i = 0
|
| 159 |
+
while i < len(word) - 1:
|
| 160 |
+
if (word[i], word[i + 1]) == most_common_pair:
|
| 161 |
+
# We found our pair! Merge it
|
| 162 |
+
new_word.append(token_bytes)
|
| 163 |
+
i += 2
|
| 164 |
+
else:
|
| 165 |
+
new_word.append(word[i])
|
| 166 |
+
i += 1
|
| 167 |
+
if i == len(word) - 1:
|
| 168 |
+
new_word.append(word[i])
|
| 169 |
+
new_words.append(new_word)
|
| 170 |
+
words = new_words
|
| 171 |
+
|
| 172 |
+
# See the intermediate merges play out!
|
| 173 |
+
if visualise:
|
| 174 |
+
print(f"The current most common pair is {most_common_pair[0]} + {most_common_pair[1]}")
|
| 175 |
+
print(f"So we made {token_bytes} our {len(ranks)}th token")
|
| 176 |
+
if visualise in ["colour", "color"]:
|
| 177 |
+
print("Now the first fifty words in our training data look like:")
|
| 178 |
+
visualise_tokens([token for word in words[:50] for token in word])
|
| 179 |
+
elif visualise == "simple":
|
| 180 |
+
print("Now the first twenty words in our training data look like:")
|
| 181 |
+
for word in words[:20]:
|
| 182 |
+
print(word)
|
| 183 |
+
print("\n")
|
| 184 |
+
|
| 185 |
+
return ranks
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def visualise_tokens(token_values: list[bytes]) -> None:
|
| 189 |
+
background = [f"\u001b[48;5;{i}m" for i in [167, 179, 185, 77, 80, 68, 134]]
|
| 190 |
+
# If token boundaries do not occur at unicode character boundaries, it's unclear how best to
|
| 191 |
+
# visualise the token. Here, we'll just use the unicode replacement character to represent some
|
| 192 |
+
# fraction of a character.
|
| 193 |
+
unicode_token_values = [x.decode("utf-8", errors="replace") for x in token_values]
|
| 194 |
+
|
| 195 |
+
running_length = 0
|
| 196 |
+
last_color = None
|
| 197 |
+
for token in unicode_token_values:
|
| 198 |
+
color = background[running_length % len(background)]
|
| 199 |
+
if color == last_color:
|
| 200 |
+
color = background[(running_length + 1) % len(background)]
|
| 201 |
+
assert color != last_color
|
| 202 |
+
last_color = color
|
| 203 |
+
running_length += len(token)
|
| 204 |
+
print(color + token, end="")
|
| 205 |
+
print("\u001b[0m")
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def train_simple_encoding():
|
| 209 |
+
gpt2_pattern = (
|
| 210 |
+
r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
|
| 211 |
+
)
|
| 212 |
+
with open(__file__) as f:
|
| 213 |
+
data = f.read()
|
| 214 |
+
|
| 215 |
+
enc = SimpleBytePairEncoding.train(data, vocab_size=600, pat_str=gpt2_pattern)
|
| 216 |
+
|
| 217 |
+
print("This is the sequence of merges performed in order to encode 'hello world':")
|
| 218 |
+
tokens = enc.encode("hello world")
|
| 219 |
+
assert enc.decode(tokens) == "hello world"
|
| 220 |
+
assert enc.decode_bytes(tokens) == b"hello world"
|
| 221 |
+
assert enc.decode_tokens_bytes(tokens) == [b"hello", b" world"]
|
| 222 |
+
|
| 223 |
+
return enc
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3a3bc5da9b8838f77f7f1441e310fb6734f29d294111de25911271f3709e298e
|
| 3 |
+
size 2307584
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/core.py
ADDED
|
@@ -0,0 +1,446 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import functools
|
| 4 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 5 |
+
from typing import TYPE_CHECKING, AbstractSet, Collection, Literal, NoReturn, Sequence
|
| 6 |
+
|
| 7 |
+
from tiktoken import _tiktoken
|
| 8 |
+
|
| 9 |
+
if TYPE_CHECKING:
|
| 10 |
+
import re
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import numpy.typing as npt
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Encoding:
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
name: str,
|
| 20 |
+
*,
|
| 21 |
+
pat_str: str,
|
| 22 |
+
mergeable_ranks: dict[bytes, int],
|
| 23 |
+
special_tokens: dict[str, int],
|
| 24 |
+
explicit_n_vocab: int | None = None,
|
| 25 |
+
):
|
| 26 |
+
"""Creates an Encoding object.
|
| 27 |
+
|
| 28 |
+
See openai_public.py for examples of how to construct an Encoding object.
|
| 29 |
+
|
| 30 |
+
Args:
|
| 31 |
+
name: The name of the encoding. It should be clear from the name of the encoding
|
| 32 |
+
what behaviour to expect, in particular, encodings with different special tokens
|
| 33 |
+
should have different names.
|
| 34 |
+
pat_str: A regex pattern string that is used to split the input text.
|
| 35 |
+
mergeable_ranks: A dictionary mapping mergeable token bytes to their ranks. The ranks
|
| 36 |
+
must correspond to merge priority.
|
| 37 |
+
special_tokens: A dictionary mapping special token strings to their token values.
|
| 38 |
+
explicit_n_vocab: The number of tokens in the vocabulary. If provided, it is checked
|
| 39 |
+
that the number of mergeable tokens and special tokens is equal to this number.
|
| 40 |
+
"""
|
| 41 |
+
self.name = name
|
| 42 |
+
|
| 43 |
+
self._pat_str = pat_str
|
| 44 |
+
self._mergeable_ranks = mergeable_ranks
|
| 45 |
+
self._special_tokens = special_tokens
|
| 46 |
+
|
| 47 |
+
self.max_token_value = max(
|
| 48 |
+
max(mergeable_ranks.values()), max(special_tokens.values(), default=0)
|
| 49 |
+
)
|
| 50 |
+
if explicit_n_vocab:
|
| 51 |
+
assert len(mergeable_ranks) + len(special_tokens) == explicit_n_vocab
|
| 52 |
+
assert self.max_token_value == explicit_n_vocab - 1
|
| 53 |
+
|
| 54 |
+
self._core_bpe = _tiktoken.CoreBPE(mergeable_ranks, special_tokens, pat_str)
|
| 55 |
+
|
| 56 |
+
def __repr__(self) -> str:
|
| 57 |
+
return f"<Encoding {self.name!r}>"
|
| 58 |
+
|
| 59 |
+
# ====================
|
| 60 |
+
# Encoding
|
| 61 |
+
# ====================
|
| 62 |
+
|
| 63 |
+
def encode_ordinary(self, text: str) -> list[int]:
|
| 64 |
+
"""Encodes a string into tokens, ignoring special tokens.
|
| 65 |
+
|
| 66 |
+
This is equivalent to `encode(text, disallowed_special=())` (but slightly faster).
|
| 67 |
+
|
| 68 |
+
```
|
| 69 |
+
>>> enc.encode_ordinary("hello world")
|
| 70 |
+
[31373, 995]
|
| 71 |
+
"""
|
| 72 |
+
try:
|
| 73 |
+
return self._core_bpe.encode_ordinary(text)
|
| 74 |
+
except UnicodeEncodeError:
|
| 75 |
+
# See comment in encode
|
| 76 |
+
text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
|
| 77 |
+
return self._core_bpe.encode_ordinary(text)
|
| 78 |
+
|
| 79 |
+
def encode(
|
| 80 |
+
self,
|
| 81 |
+
text: str,
|
| 82 |
+
*,
|
| 83 |
+
allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
|
| 84 |
+
disallowed_special: Literal["all"] | Collection[str] = "all",
|
| 85 |
+
) -> list[int]:
|
| 86 |
+
"""Encodes a string into tokens.
|
| 87 |
+
|
| 88 |
+
Special tokens are artificial tokens used to unlock capabilities from a model,
|
| 89 |
+
such as fill-in-the-middle. So we want to be careful about accidentally encoding special
|
| 90 |
+
tokens, since they can be used to trick a model into doing something we don't want it to do.
|
| 91 |
+
|
| 92 |
+
Hence, by default, encode will raise an error if it encounters text that corresponds
|
| 93 |
+
to a special token. This can be controlled on a per-token level using the `allowed_special`
|
| 94 |
+
and `disallowed_special` parameters. In particular:
|
| 95 |
+
- Setting `disallowed_special` to () will prevent this function from raising errors and
|
| 96 |
+
cause all text corresponding to special tokens to be encoded as natural text.
|
| 97 |
+
- Setting `allowed_special` to "all" will cause this function to treat all text
|
| 98 |
+
corresponding to special tokens to be encoded as special tokens.
|
| 99 |
+
|
| 100 |
+
```
|
| 101 |
+
>>> enc.encode("hello world")
|
| 102 |
+
[31373, 995]
|
| 103 |
+
>>> enc.encode("<|endoftext|>", allowed_special={"<|endoftext|>"})
|
| 104 |
+
[50256]
|
| 105 |
+
>>> enc.encode("<|endoftext|>", allowed_special="all")
|
| 106 |
+
[50256]
|
| 107 |
+
>>> enc.encode("<|endoftext|>")
|
| 108 |
+
# Raises ValueError
|
| 109 |
+
>>> enc.encode("<|endoftext|>", disallowed_special=())
|
| 110 |
+
[27, 91, 437, 1659, 5239, 91, 29]
|
| 111 |
+
```
|
| 112 |
+
"""
|
| 113 |
+
if allowed_special == "all":
|
| 114 |
+
allowed_special = self.special_tokens_set
|
| 115 |
+
if disallowed_special == "all":
|
| 116 |
+
disallowed_special = self.special_tokens_set - allowed_special
|
| 117 |
+
if disallowed_special:
|
| 118 |
+
if not isinstance(disallowed_special, frozenset):
|
| 119 |
+
disallowed_special = frozenset(disallowed_special)
|
| 120 |
+
if match := _special_token_regex(disallowed_special).search(text):
|
| 121 |
+
raise_disallowed_special_token(match.group())
|
| 122 |
+
|
| 123 |
+
try:
|
| 124 |
+
return self._core_bpe.encode(text, allowed_special)
|
| 125 |
+
except UnicodeEncodeError:
|
| 126 |
+
# BPE operates on bytes, but the regex operates on unicode. If we pass a str that is
|
| 127 |
+
# invalid UTF-8 to Rust, it will rightfully complain. Here we do a quick and dirty
|
| 128 |
+
# fixup for any surrogate pairs that may have sneaked their way into the text.
|
| 129 |
+
# Technically, this introduces a place where encode + decode doesn't roundtrip a Python
|
| 130 |
+
# string, but given that this is input we want to support, maybe that's okay.
|
| 131 |
+
# Also we use errors="replace" to handle weird things like lone surrogates.
|
| 132 |
+
text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
|
| 133 |
+
return self._core_bpe.encode(text, allowed_special)
|
| 134 |
+
|
| 135 |
+
def encode_to_numpy(
|
| 136 |
+
self,
|
| 137 |
+
text: str,
|
| 138 |
+
*,
|
| 139 |
+
allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
|
| 140 |
+
disallowed_special: Literal["all"] | Collection[str] = "all",
|
| 141 |
+
) -> npt.NDArray[np.uint32]:
|
| 142 |
+
"""Encodes a string into tokens, returning a numpy array.
|
| 143 |
+
|
| 144 |
+
Avoids the overhead of copying the token buffer into a Python list.
|
| 145 |
+
"""
|
| 146 |
+
if allowed_special == "all":
|
| 147 |
+
allowed_special = self.special_tokens_set
|
| 148 |
+
if disallowed_special == "all":
|
| 149 |
+
disallowed_special = self.special_tokens_set - allowed_special
|
| 150 |
+
if disallowed_special:
|
| 151 |
+
if not isinstance(disallowed_special, frozenset):
|
| 152 |
+
disallowed_special = frozenset(disallowed_special)
|
| 153 |
+
if match := _special_token_regex(disallowed_special).search(text):
|
| 154 |
+
raise_disallowed_special_token(match.group())
|
| 155 |
+
|
| 156 |
+
import numpy as np
|
| 157 |
+
|
| 158 |
+
buffer = self._core_bpe.encode_to_tiktoken_buffer(text, allowed_special)
|
| 159 |
+
return np.frombuffer(buffer, dtype=np.uint32)
|
| 160 |
+
|
| 161 |
+
def encode_ordinary_batch(self, text: list[str], *, num_threads: int = 8) -> list[list[int]]:
|
| 162 |
+
"""Encodes a list of strings into tokens, in parallel, ignoring special tokens.
|
| 163 |
+
|
| 164 |
+
This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster).
|
| 165 |
+
|
| 166 |
+
```
|
| 167 |
+
>>> enc.encode_ordinary_batch(["hello world", "goodbye world"])
|
| 168 |
+
[[31373, 995], [11274, 16390, 995]]
|
| 169 |
+
```
|
| 170 |
+
"""
|
| 171 |
+
encoder = functools.partial(self.encode_ordinary)
|
| 172 |
+
with ThreadPoolExecutor(num_threads) as e:
|
| 173 |
+
return list(e.map(encoder, text))
|
| 174 |
+
|
| 175 |
+
def encode_batch(
|
| 176 |
+
self,
|
| 177 |
+
text: list[str],
|
| 178 |
+
*,
|
| 179 |
+
num_threads: int = 8,
|
| 180 |
+
allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
|
| 181 |
+
disallowed_special: Literal["all"] | Collection[str] = "all",
|
| 182 |
+
) -> list[list[int]]:
|
| 183 |
+
"""Encodes a list of strings into tokens, in parallel.
|
| 184 |
+
|
| 185 |
+
See `encode` for more details on `allowed_special` and `disallowed_special`.
|
| 186 |
+
|
| 187 |
+
```
|
| 188 |
+
>>> enc.encode_batch(["hello world", "goodbye world"])
|
| 189 |
+
[[31373, 995], [11274, 16390, 995]]
|
| 190 |
+
```
|
| 191 |
+
"""
|
| 192 |
+
if allowed_special == "all":
|
| 193 |
+
allowed_special = self.special_tokens_set
|
| 194 |
+
if disallowed_special == "all":
|
| 195 |
+
disallowed_special = self.special_tokens_set - allowed_special
|
| 196 |
+
if not isinstance(disallowed_special, frozenset):
|
| 197 |
+
disallowed_special = frozenset(disallowed_special)
|
| 198 |
+
|
| 199 |
+
encoder = functools.partial(
|
| 200 |
+
self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special
|
| 201 |
+
)
|
| 202 |
+
with ThreadPoolExecutor(num_threads) as e:
|
| 203 |
+
return list(e.map(encoder, text))
|
| 204 |
+
|
| 205 |
+
def encode_with_unstable(
|
| 206 |
+
self,
|
| 207 |
+
text: str,
|
| 208 |
+
*,
|
| 209 |
+
allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
|
| 210 |
+
disallowed_special: Literal["all"] | Collection[str] = "all",
|
| 211 |
+
) -> tuple[list[int], list[list[int]]]:
|
| 212 |
+
"""Encodes a string into stable tokens and possible completion sequences.
|
| 213 |
+
|
| 214 |
+
Note that the stable tokens will only represent a substring of `text`.
|
| 215 |
+
|
| 216 |
+
See `encode` for more details on `allowed_special` and `disallowed_special`.
|
| 217 |
+
|
| 218 |
+
This API should itself be considered unstable.
|
| 219 |
+
|
| 220 |
+
```
|
| 221 |
+
>>> enc.encode_with_unstable("hello fanta")
|
| 222 |
+
([31373], [(277, 4910), (5113, 265), ..., (8842,)])
|
| 223 |
+
|
| 224 |
+
>>> text = "..."
|
| 225 |
+
>>> stable_tokens, completions = enc.encode_with_unstable(text)
|
| 226 |
+
>>> assert text.encode().startswith(enc.decode_bytes(stable_tokens))
|
| 227 |
+
>>> assert all(enc.decode_bytes(stable_tokens + seq).startswith(text.encode()) for seq in completions)
|
| 228 |
+
```
|
| 229 |
+
"""
|
| 230 |
+
if allowed_special == "all":
|
| 231 |
+
allowed_special = self.special_tokens_set
|
| 232 |
+
if disallowed_special == "all":
|
| 233 |
+
disallowed_special = self.special_tokens_set - allowed_special
|
| 234 |
+
if disallowed_special:
|
| 235 |
+
if not isinstance(disallowed_special, frozenset):
|
| 236 |
+
disallowed_special = frozenset(disallowed_special)
|
| 237 |
+
if match := _special_token_regex(disallowed_special).search(text):
|
| 238 |
+
raise_disallowed_special_token(match.group())
|
| 239 |
+
|
| 240 |
+
return self._core_bpe.encode_with_unstable(text, allowed_special)
|
| 241 |
+
|
| 242 |
+
def encode_single_token(self, text_or_bytes: str | bytes) -> int:
|
| 243 |
+
"""Encodes text corresponding to a single token to its token value.
|
| 244 |
+
|
| 245 |
+
NOTE: this will encode all special tokens.
|
| 246 |
+
|
| 247 |
+
Raises `KeyError` if the token is not in the vocabulary.
|
| 248 |
+
|
| 249 |
+
```
|
| 250 |
+
>>> enc.encode_single_token("hello")
|
| 251 |
+
31373
|
| 252 |
+
```
|
| 253 |
+
"""
|
| 254 |
+
if isinstance(text_or_bytes, str):
|
| 255 |
+
text_or_bytes = text_or_bytes.encode("utf-8")
|
| 256 |
+
return self._core_bpe.encode_single_token(text_or_bytes)
|
| 257 |
+
|
| 258 |
+
# ====================
|
| 259 |
+
# Decoding
|
| 260 |
+
# ====================
|
| 261 |
+
|
| 262 |
+
def decode_bytes(self, tokens: Sequence[int]) -> bytes:
|
| 263 |
+
"""Decodes a list of tokens into bytes.
|
| 264 |
+
|
| 265 |
+
```
|
| 266 |
+
>>> enc.decode_bytes([31373, 995])
|
| 267 |
+
b'hello world'
|
| 268 |
+
```
|
| 269 |
+
"""
|
| 270 |
+
return self._core_bpe.decode_bytes(tokens)
|
| 271 |
+
|
| 272 |
+
def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
|
| 273 |
+
"""Decodes a list of tokens into a string.
|
| 274 |
+
|
| 275 |
+
WARNING: the default behaviour of this function is lossy, since decoded bytes are not
|
| 276 |
+
guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter,
|
| 277 |
+
for instance, setting `errors=strict`.
|
| 278 |
+
|
| 279 |
+
```
|
| 280 |
+
>>> enc.decode([31373, 995])
|
| 281 |
+
'hello world'
|
| 282 |
+
```
|
| 283 |
+
"""
|
| 284 |
+
return self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors)
|
| 285 |
+
|
| 286 |
+
def decode_single_token_bytes(self, token: int) -> bytes:
|
| 287 |
+
"""Decodes a token into bytes.
|
| 288 |
+
|
| 289 |
+
NOTE: this will decode all special tokens.
|
| 290 |
+
|
| 291 |
+
Raises `KeyError` if the token is not in the vocabulary.
|
| 292 |
+
|
| 293 |
+
```
|
| 294 |
+
>>> enc.decode_single_token_bytes(31373)
|
| 295 |
+
b'hello'
|
| 296 |
+
```
|
| 297 |
+
"""
|
| 298 |
+
return self._core_bpe.decode_single_token_bytes(token)
|
| 299 |
+
|
| 300 |
+
def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]:
|
| 301 |
+
"""Decodes a list of tokens into a list of bytes.
|
| 302 |
+
|
| 303 |
+
Useful for visualising tokenisation.
|
| 304 |
+
>>> enc.decode_tokens_bytes([31373, 995])
|
| 305 |
+
[b'hello', b' world']
|
| 306 |
+
"""
|
| 307 |
+
return [self.decode_single_token_bytes(token) for token in tokens]
|
| 308 |
+
|
| 309 |
+
def decode_with_offsets(self, tokens: Sequence[int]) -> tuple[str, list[int]]:
|
| 310 |
+
"""Decodes a list of tokens into a string and a list of offsets.
|
| 311 |
+
|
| 312 |
+
Each offset is the index into text corresponding to the start of each token.
|
| 313 |
+
If UTF-8 character boundaries do not line up with token boundaries, the offset is the index
|
| 314 |
+
of the first character that contains bytes from the token.
|
| 315 |
+
|
| 316 |
+
This will currently raise if given tokens that decode to invalid UTF-8; this behaviour may
|
| 317 |
+
change in the future to be more permissive.
|
| 318 |
+
|
| 319 |
+
>>> enc.decode_with_offsets([31373, 995])
|
| 320 |
+
('hello world', [0, 5])
|
| 321 |
+
"""
|
| 322 |
+
token_bytes = self.decode_tokens_bytes(tokens)
|
| 323 |
+
|
| 324 |
+
text_len = 0
|
| 325 |
+
offsets = []
|
| 326 |
+
for token in token_bytes:
|
| 327 |
+
offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0)))
|
| 328 |
+
text_len += sum(1 for c in token if not 0x80 <= c < 0xC0)
|
| 329 |
+
|
| 330 |
+
# TODO: assess correctness for errors="ignore" and errors="replace"
|
| 331 |
+
text = b"".join(token_bytes).decode("utf-8", errors="strict")
|
| 332 |
+
return text, offsets
|
| 333 |
+
|
| 334 |
+
def decode_batch(
|
| 335 |
+
self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
|
| 336 |
+
) -> list[str]:
|
| 337 |
+
"""Decodes a batch (list of lists of tokens) into a list of strings."""
|
| 338 |
+
decoder = functools.partial(self.decode, errors=errors)
|
| 339 |
+
with ThreadPoolExecutor(num_threads) as e:
|
| 340 |
+
return list(e.map(decoder, batch))
|
| 341 |
+
|
| 342 |
+
def decode_bytes_batch(
|
| 343 |
+
self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
|
| 344 |
+
) -> list[bytes]:
|
| 345 |
+
"""Decodes a batch (list of lists of tokens) into a list of bytes."""
|
| 346 |
+
with ThreadPoolExecutor(num_threads) as e:
|
| 347 |
+
return list(e.map(self.decode_bytes, batch))
|
| 348 |
+
|
| 349 |
+
# ====================
|
| 350 |
+
# Miscellaneous
|
| 351 |
+
# ====================
|
| 352 |
+
|
| 353 |
+
def token_byte_values(self) -> list[bytes]:
|
| 354 |
+
"""Returns the list of all token byte values."""
|
| 355 |
+
return self._core_bpe.token_byte_values()
|
| 356 |
+
|
| 357 |
+
@property
|
| 358 |
+
def eot_token(self) -> int:
|
| 359 |
+
return self._special_tokens["<|endoftext|>"]
|
| 360 |
+
|
| 361 |
+
@functools.cached_property
|
| 362 |
+
def special_tokens_set(self) -> set[str]:
|
| 363 |
+
return set(self._special_tokens.keys())
|
| 364 |
+
|
| 365 |
+
def is_special_token(self, token: int) -> bool:
|
| 366 |
+
assert isinstance(token, int)
|
| 367 |
+
return token in self._special_token_values
|
| 368 |
+
|
| 369 |
+
@property
|
| 370 |
+
def n_vocab(self) -> int:
|
| 371 |
+
"""For backwards compatibility. Prefer to use `enc.max_token_value + 1`."""
|
| 372 |
+
return self.max_token_value + 1
|
| 373 |
+
|
| 374 |
+
# ====================
|
| 375 |
+
# Private
|
| 376 |
+
# ====================
|
| 377 |
+
|
| 378 |
+
def _encode_single_piece(self, text_or_bytes: str | bytes) -> list[int]:
|
| 379 |
+
"""Encodes text corresponding to bytes without a regex split.
|
| 380 |
+
|
| 381 |
+
NOTE: this will not encode any special tokens.
|
| 382 |
+
|
| 383 |
+
```
|
| 384 |
+
>>> enc.encode_single_piece("helloqqqq")
|
| 385 |
+
[31373, 38227, 38227]
|
| 386 |
+
```
|
| 387 |
+
"""
|
| 388 |
+
if isinstance(text_or_bytes, str):
|
| 389 |
+
text_or_bytes = text_or_bytes.encode("utf-8")
|
| 390 |
+
return self._core_bpe.encode_single_piece(text_or_bytes)
|
| 391 |
+
|
| 392 |
+
def _encode_only_native_bpe(self, text: str) -> list[int]:
|
| 393 |
+
"""Encodes a string into tokens, but do regex splitting in Python."""
|
| 394 |
+
# We need specifically `regex` in order to compile pat_str due to e.g. \p
|
| 395 |
+
import regex
|
| 396 |
+
|
| 397 |
+
_unused_pat = regex.compile(self._pat_str)
|
| 398 |
+
ret = []
|
| 399 |
+
for piece in regex.findall(_unused_pat, text):
|
| 400 |
+
ret.extend(self._core_bpe.encode_single_piece(piece.encode("utf-8")))
|
| 401 |
+
return ret
|
| 402 |
+
|
| 403 |
+
def _encode_bytes(self, text: bytes) -> list[int]:
|
| 404 |
+
return self._core_bpe._encode_bytes(text)
|
| 405 |
+
|
| 406 |
+
def __getstate__(self) -> object:
|
| 407 |
+
import tiktoken.registry
|
| 408 |
+
|
| 409 |
+
# As an optimisation, pickle registered encodings by reference
|
| 410 |
+
if self is tiktoken.registry.ENCODINGS.get(self.name):
|
| 411 |
+
return self.name
|
| 412 |
+
return {
|
| 413 |
+
"name": self.name,
|
| 414 |
+
"pat_str": self._pat_str,
|
| 415 |
+
"mergeable_ranks": self._mergeable_ranks,
|
| 416 |
+
"special_tokens": self._special_tokens,
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
def __setstate__(self, value: object) -> None:
|
| 420 |
+
import tiktoken.registry
|
| 421 |
+
|
| 422 |
+
if isinstance(value, str):
|
| 423 |
+
self.__dict__ = tiktoken.registry.get_encoding(value).__dict__
|
| 424 |
+
return
|
| 425 |
+
self.__init__(**value)
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
@functools.lru_cache(maxsize=128)
|
| 429 |
+
def _special_token_regex(tokens: frozenset[str]) -> re.Pattern[str]:
|
| 430 |
+
try:
|
| 431 |
+
import regex as re
|
| 432 |
+
except ImportError:
|
| 433 |
+
import re
|
| 434 |
+
inner = "|".join(re.escape(token) for token in tokens)
|
| 435 |
+
return re.compile(f"({inner})")
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def raise_disallowed_special_token(token: str) -> NoReturn:
|
| 439 |
+
raise ValueError(
|
| 440 |
+
f"Encountered text corresponding to disallowed special token {token!r}.\n"
|
| 441 |
+
"If you want this text to be encoded as a special token, "
|
| 442 |
+
f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"
|
| 443 |
+
f"If you want this text to be encoded as normal text, disable the check for this token "
|
| 444 |
+
f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n"
|
| 445 |
+
"To disable this check for all special tokens, pass `disallowed_special=()`.\n"
|
| 446 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/load.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import base64
|
| 4 |
+
import hashlib
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def read_file(blobpath: str) -> bytes:
|
| 9 |
+
if "://" not in blobpath:
|
| 10 |
+
with open(blobpath, "rb", buffering=0) as f:
|
| 11 |
+
return f.read()
|
| 12 |
+
|
| 13 |
+
if blobpath.startswith(("http://", "https://")):
|
| 14 |
+
# avoiding blobfile for public files helps avoid auth issues, like MFA prompts.
|
| 15 |
+
import requests
|
| 16 |
+
|
| 17 |
+
resp = requests.get(blobpath)
|
| 18 |
+
resp.raise_for_status()
|
| 19 |
+
return resp.content
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
import blobfile
|
| 23 |
+
except ImportError as e:
|
| 24 |
+
raise ImportError(
|
| 25 |
+
"blobfile is not installed. Please install it by running `pip install blobfile`."
|
| 26 |
+
) from e
|
| 27 |
+
with blobfile.BlobFile(blobpath, "rb") as f:
|
| 28 |
+
return f.read()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def check_hash(data: bytes, expected_hash: str) -> bool:
|
| 32 |
+
actual_hash = hashlib.sha256(data).hexdigest()
|
| 33 |
+
return actual_hash == expected_hash
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def read_file_cached(blobpath: str, expected_hash: str | None = None) -> bytes:
|
| 37 |
+
user_specified_cache = True
|
| 38 |
+
if "TIKTOKEN_CACHE_DIR" in os.environ:
|
| 39 |
+
cache_dir = os.environ["TIKTOKEN_CACHE_DIR"]
|
| 40 |
+
elif "DATA_GYM_CACHE_DIR" in os.environ:
|
| 41 |
+
cache_dir = os.environ["DATA_GYM_CACHE_DIR"]
|
| 42 |
+
else:
|
| 43 |
+
import tempfile
|
| 44 |
+
|
| 45 |
+
cache_dir = os.path.join(tempfile.gettempdir(), "data-gym-cache")
|
| 46 |
+
user_specified_cache = False
|
| 47 |
+
|
| 48 |
+
if cache_dir == "":
|
| 49 |
+
# disable caching
|
| 50 |
+
return read_file(blobpath)
|
| 51 |
+
|
| 52 |
+
cache_key = hashlib.sha1(blobpath.encode()).hexdigest()
|
| 53 |
+
|
| 54 |
+
cache_path = os.path.join(cache_dir, cache_key)
|
| 55 |
+
if os.path.exists(cache_path):
|
| 56 |
+
with open(cache_path, "rb", buffering=0) as f:
|
| 57 |
+
data = f.read()
|
| 58 |
+
if expected_hash is None or check_hash(data, expected_hash):
|
| 59 |
+
return data
|
| 60 |
+
|
| 61 |
+
# the cached file does not match the hash, remove it and re-fetch
|
| 62 |
+
try:
|
| 63 |
+
os.remove(cache_path)
|
| 64 |
+
except OSError:
|
| 65 |
+
pass
|
| 66 |
+
|
| 67 |
+
contents = read_file(blobpath)
|
| 68 |
+
if expected_hash and not check_hash(contents, expected_hash):
|
| 69 |
+
raise ValueError(
|
| 70 |
+
f"Hash mismatch for data downloaded from {blobpath} (expected {expected_hash}). "
|
| 71 |
+
f"This may indicate a corrupted download. Please try again."
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
import uuid
|
| 75 |
+
|
| 76 |
+
try:
|
| 77 |
+
os.makedirs(cache_dir, exist_ok=True)
|
| 78 |
+
tmp_filename = cache_path + "." + str(uuid.uuid4()) + ".tmp"
|
| 79 |
+
with open(tmp_filename, "wb") as f:
|
| 80 |
+
f.write(contents)
|
| 81 |
+
os.rename(tmp_filename, cache_path)
|
| 82 |
+
except OSError:
|
| 83 |
+
# don't raise if we can't write to the default cache, e.g. issue #75
|
| 84 |
+
if user_specified_cache:
|
| 85 |
+
raise
|
| 86 |
+
|
| 87 |
+
return contents
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def data_gym_to_mergeable_bpe_ranks(
|
| 91 |
+
vocab_bpe_file: str,
|
| 92 |
+
encoder_json_file: str,
|
| 93 |
+
vocab_bpe_hash: str | None = None,
|
| 94 |
+
encoder_json_hash: str | None = None,
|
| 95 |
+
clobber_one_byte_tokens: bool = False,
|
| 96 |
+
) -> dict[bytes, int]:
|
| 97 |
+
# NB: do not add caching to this function
|
| 98 |
+
rank_to_intbyte = [b for b in range(2**8) if chr(b).isprintable() and chr(b) != " "]
|
| 99 |
+
|
| 100 |
+
data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte}
|
| 101 |
+
n = 0
|
| 102 |
+
for b in range(2**8):
|
| 103 |
+
if b not in rank_to_intbyte:
|
| 104 |
+
rank_to_intbyte.append(b)
|
| 105 |
+
data_gym_byte_to_byte[chr(2**8 + n)] = b
|
| 106 |
+
n += 1
|
| 107 |
+
assert len(rank_to_intbyte) == 2**8
|
| 108 |
+
|
| 109 |
+
# vocab_bpe contains the merges along with associated ranks
|
| 110 |
+
vocab_bpe_contents = read_file_cached(vocab_bpe_file, vocab_bpe_hash).decode()
|
| 111 |
+
bpe_merges = [tuple(merge_str.split()) for merge_str in vocab_bpe_contents.split("\n")[1:-1]]
|
| 112 |
+
|
| 113 |
+
def decode_data_gym(value: str) -> bytes:
|
| 114 |
+
return bytes(data_gym_byte_to_byte[b] for b in value)
|
| 115 |
+
|
| 116 |
+
# add the single byte tokens
|
| 117 |
+
# if clobber_one_byte_tokens is True, we'll replace these with ones from the encoder json
|
| 118 |
+
bpe_ranks = {bytes([b]): i for i, b in enumerate(rank_to_intbyte)}
|
| 119 |
+
del rank_to_intbyte
|
| 120 |
+
|
| 121 |
+
# add the merged tokens
|
| 122 |
+
n = len(bpe_ranks)
|
| 123 |
+
for first, second in bpe_merges:
|
| 124 |
+
bpe_ranks[decode_data_gym(first) + decode_data_gym(second)] = n
|
| 125 |
+
n += 1
|
| 126 |
+
|
| 127 |
+
import json
|
| 128 |
+
|
| 129 |
+
# check that the encoder file matches the merges file
|
| 130 |
+
# this sanity check is important since tiktoken assumes that ranks are ordered the same
|
| 131 |
+
# as merge priority
|
| 132 |
+
encoder_json = json.loads(read_file_cached(encoder_json_file, encoder_json_hash))
|
| 133 |
+
encoder_json_loaded = {decode_data_gym(k): v for k, v in encoder_json.items()}
|
| 134 |
+
# drop these two special tokens if present, since they're not mergeable bpe tokens
|
| 135 |
+
encoder_json_loaded.pop(b"<|endoftext|>", None)
|
| 136 |
+
encoder_json_loaded.pop(b"<|startoftext|>", None)
|
| 137 |
+
|
| 138 |
+
if clobber_one_byte_tokens:
|
| 139 |
+
for k in encoder_json_loaded:
|
| 140 |
+
if len(k) == 1:
|
| 141 |
+
bpe_ranks[k] = encoder_json_loaded[k]
|
| 142 |
+
|
| 143 |
+
assert bpe_ranks == encoder_json_loaded
|
| 144 |
+
|
| 145 |
+
return bpe_ranks
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def dump_tiktoken_bpe(bpe_ranks: dict[bytes, int], tiktoken_bpe_file: str) -> None:
|
| 149 |
+
try:
|
| 150 |
+
import blobfile
|
| 151 |
+
except ImportError as e:
|
| 152 |
+
raise ImportError(
|
| 153 |
+
"blobfile is not installed. Please install it by running `pip install blobfile`."
|
| 154 |
+
) from e
|
| 155 |
+
with blobfile.BlobFile(tiktoken_bpe_file, "wb") as f:
|
| 156 |
+
for token, rank in sorted(bpe_ranks.items(), key=lambda x: x[1]):
|
| 157 |
+
f.write(base64.b64encode(token) + b" " + str(rank).encode() + b"\n")
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def load_tiktoken_bpe(tiktoken_bpe_file: str, expected_hash: str | None = None) -> dict[bytes, int]:
|
| 161 |
+
# NB: do not add caching to this function
|
| 162 |
+
contents = read_file_cached(tiktoken_bpe_file, expected_hash)
|
| 163 |
+
ret = {}
|
| 164 |
+
for line in contents.splitlines():
|
| 165 |
+
if not line:
|
| 166 |
+
continue
|
| 167 |
+
try:
|
| 168 |
+
token, rank = line.split()
|
| 169 |
+
ret[base64.b64decode(token)] = int(rank)
|
| 170 |
+
except Exception as e:
|
| 171 |
+
raise ValueError(f"Error parsing line {line!r} in {tiktoken_bpe_file}") from e
|
| 172 |
+
return ret
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/model.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from .core import Encoding
|
| 4 |
+
from .registry import get_encoding
|
| 5 |
+
|
| 6 |
+
# TODO: these will likely be replaced by an API endpoint
|
| 7 |
+
MODEL_PREFIX_TO_ENCODING: dict[str, str] = {
|
| 8 |
+
"o1-": "o200k_base",
|
| 9 |
+
"o3-": "o200k_base",
|
| 10 |
+
"o4-mini-": "o200k_base",
|
| 11 |
+
# chat
|
| 12 |
+
"gpt-5-": "o200k_base",
|
| 13 |
+
"gpt-4.5-": "o200k_base",
|
| 14 |
+
"gpt-4.1-": "o200k_base",
|
| 15 |
+
"chatgpt-4o-": "o200k_base",
|
| 16 |
+
"gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13
|
| 17 |
+
"gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k
|
| 18 |
+
"gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc.
|
| 19 |
+
"gpt-35-turbo-": "cl100k_base", # Azure deployment name
|
| 20 |
+
"gpt-oss-": "o200k_harmony",
|
| 21 |
+
# fine-tuned
|
| 22 |
+
"ft:gpt-4o": "o200k_base",
|
| 23 |
+
"ft:gpt-4": "cl100k_base",
|
| 24 |
+
"ft:gpt-3.5-turbo": "cl100k_base",
|
| 25 |
+
"ft:davinci-002": "cl100k_base",
|
| 26 |
+
"ft:babbage-002": "cl100k_base",
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
MODEL_TO_ENCODING: dict[str, str] = {
|
| 30 |
+
# reasoning
|
| 31 |
+
"o1": "o200k_base",
|
| 32 |
+
"o3": "o200k_base",
|
| 33 |
+
"o4-mini": "o200k_base",
|
| 34 |
+
# chat
|
| 35 |
+
"gpt-5": "o200k_base",
|
| 36 |
+
"gpt-4.1": "o200k_base",
|
| 37 |
+
"gpt-4o": "o200k_base",
|
| 38 |
+
"gpt-4": "cl100k_base",
|
| 39 |
+
"gpt-3.5-turbo": "cl100k_base",
|
| 40 |
+
"gpt-3.5": "cl100k_base", # Common shorthand
|
| 41 |
+
"gpt-35-turbo": "cl100k_base", # Azure deployment name
|
| 42 |
+
# base
|
| 43 |
+
"davinci-002": "cl100k_base",
|
| 44 |
+
"babbage-002": "cl100k_base",
|
| 45 |
+
# embeddings
|
| 46 |
+
"text-embedding-ada-002": "cl100k_base",
|
| 47 |
+
"text-embedding-3-small": "cl100k_base",
|
| 48 |
+
"text-embedding-3-large": "cl100k_base",
|
| 49 |
+
# DEPRECATED MODELS
|
| 50 |
+
# text (DEPRECATED)
|
| 51 |
+
"text-davinci-003": "p50k_base",
|
| 52 |
+
"text-davinci-002": "p50k_base",
|
| 53 |
+
"text-davinci-001": "r50k_base",
|
| 54 |
+
"text-curie-001": "r50k_base",
|
| 55 |
+
"text-babbage-001": "r50k_base",
|
| 56 |
+
"text-ada-001": "r50k_base",
|
| 57 |
+
"davinci": "r50k_base",
|
| 58 |
+
"curie": "r50k_base",
|
| 59 |
+
"babbage": "r50k_base",
|
| 60 |
+
"ada": "r50k_base",
|
| 61 |
+
# code (DEPRECATED)
|
| 62 |
+
"code-davinci-002": "p50k_base",
|
| 63 |
+
"code-davinci-001": "p50k_base",
|
| 64 |
+
"code-cushman-002": "p50k_base",
|
| 65 |
+
"code-cushman-001": "p50k_base",
|
| 66 |
+
"davinci-codex": "p50k_base",
|
| 67 |
+
"cushman-codex": "p50k_base",
|
| 68 |
+
# edit (DEPRECATED)
|
| 69 |
+
"text-davinci-edit-001": "p50k_edit",
|
| 70 |
+
"code-davinci-edit-001": "p50k_edit",
|
| 71 |
+
# old embeddings (DEPRECATED)
|
| 72 |
+
"text-similarity-davinci-001": "r50k_base",
|
| 73 |
+
"text-similarity-curie-001": "r50k_base",
|
| 74 |
+
"text-similarity-babbage-001": "r50k_base",
|
| 75 |
+
"text-similarity-ada-001": "r50k_base",
|
| 76 |
+
"text-search-davinci-doc-001": "r50k_base",
|
| 77 |
+
"text-search-curie-doc-001": "r50k_base",
|
| 78 |
+
"text-search-babbage-doc-001": "r50k_base",
|
| 79 |
+
"text-search-ada-doc-001": "r50k_base",
|
| 80 |
+
"code-search-babbage-code-001": "r50k_base",
|
| 81 |
+
"code-search-ada-code-001": "r50k_base",
|
| 82 |
+
# open source
|
| 83 |
+
"gpt2": "gpt2",
|
| 84 |
+
"gpt-2": "gpt2", # Maintains consistency with gpt-4
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def encoding_name_for_model(model_name: str) -> str:
|
| 89 |
+
"""Returns the name of the encoding used by a model.
|
| 90 |
+
|
| 91 |
+
Raises a KeyError if the model name is not recognised.
|
| 92 |
+
"""
|
| 93 |
+
encoding_name = None
|
| 94 |
+
if model_name in MODEL_TO_ENCODING:
|
| 95 |
+
encoding_name = MODEL_TO_ENCODING[model_name]
|
| 96 |
+
else:
|
| 97 |
+
# Check if the model matches a known prefix
|
| 98 |
+
# Prefix matching avoids needing library updates for every model version release
|
| 99 |
+
# Note that this can match on non-existent models (e.g., gpt-3.5-turbo-FAKE)
|
| 100 |
+
for model_prefix, model_encoding_name in MODEL_PREFIX_TO_ENCODING.items():
|
| 101 |
+
if model_name.startswith(model_prefix):
|
| 102 |
+
return model_encoding_name
|
| 103 |
+
|
| 104 |
+
if encoding_name is None:
|
| 105 |
+
raise KeyError(
|
| 106 |
+
f"Could not automatically map {model_name} to a tokeniser. "
|
| 107 |
+
"Please use `tiktoken.get_encoding` to explicitly get the tokeniser you expect."
|
| 108 |
+
) from None
|
| 109 |
+
|
| 110 |
+
return encoding_name
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def encoding_for_model(model_name: str) -> Encoding:
|
| 114 |
+
"""Returns the encoding used by a model.
|
| 115 |
+
|
| 116 |
+
Raises a KeyError if the model name is not recognised.
|
| 117 |
+
"""
|
| 118 |
+
return get_encoding(encoding_name_for_model(model_name))
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/py.typed
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/registry.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import functools
|
| 4 |
+
import importlib
|
| 5 |
+
import pkgutil
|
| 6 |
+
import threading
|
| 7 |
+
from typing import Any, Callable, Sequence
|
| 8 |
+
|
| 9 |
+
import tiktoken_ext
|
| 10 |
+
|
| 11 |
+
import tiktoken
|
| 12 |
+
from tiktoken.core import Encoding
|
| 13 |
+
|
| 14 |
+
_lock = threading.RLock()
|
| 15 |
+
ENCODINGS: dict[str, Encoding] = {}
|
| 16 |
+
ENCODING_CONSTRUCTORS: dict[str, Callable[[], dict[str, Any]]] | None = None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@functools.lru_cache
|
| 20 |
+
def _available_plugin_modules() -> Sequence[str]:
|
| 21 |
+
# tiktoken_ext is a namespace package
|
| 22 |
+
# submodules inside tiktoken_ext will be inspected for ENCODING_CONSTRUCTORS attributes
|
| 23 |
+
# - we use namespace package pattern so `pkgutil.iter_modules` is fast
|
| 24 |
+
# - it's a separate top-level package because namespace subpackages of non-namespace
|
| 25 |
+
# packages don't quite do what you want with editable installs
|
| 26 |
+
mods = []
|
| 27 |
+
plugin_mods = pkgutil.iter_modules(tiktoken_ext.__path__, tiktoken_ext.__name__ + ".")
|
| 28 |
+
for _, mod_name, _ in plugin_mods:
|
| 29 |
+
mods.append(mod_name)
|
| 30 |
+
return mods
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _find_constructors() -> None:
|
| 34 |
+
global ENCODING_CONSTRUCTORS
|
| 35 |
+
with _lock:
|
| 36 |
+
if ENCODING_CONSTRUCTORS is not None:
|
| 37 |
+
return
|
| 38 |
+
ENCODING_CONSTRUCTORS = {}
|
| 39 |
+
|
| 40 |
+
try:
|
| 41 |
+
for mod_name in _available_plugin_modules():
|
| 42 |
+
mod = importlib.import_module(mod_name)
|
| 43 |
+
try:
|
| 44 |
+
constructors = mod.ENCODING_CONSTRUCTORS
|
| 45 |
+
except AttributeError as e:
|
| 46 |
+
raise ValueError(
|
| 47 |
+
f"tiktoken plugin {mod_name} does not define ENCODING_CONSTRUCTORS"
|
| 48 |
+
) from e
|
| 49 |
+
for enc_name, constructor in constructors.items():
|
| 50 |
+
if enc_name in ENCODING_CONSTRUCTORS:
|
| 51 |
+
raise ValueError(
|
| 52 |
+
f"Duplicate encoding name {enc_name} in tiktoken plugin {mod_name}"
|
| 53 |
+
)
|
| 54 |
+
ENCODING_CONSTRUCTORS[enc_name] = constructor
|
| 55 |
+
except Exception:
|
| 56 |
+
# Ensure we idempotently raise errors
|
| 57 |
+
ENCODING_CONSTRUCTORS = None
|
| 58 |
+
raise
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def get_encoding(encoding_name: str) -> Encoding:
|
| 64 |
+
if not isinstance(encoding_name, str):
|
| 65 |
+
raise ValueError(f"Expected a string in get_encoding, got {type(encoding_name)}")
|
| 66 |
+
|
| 67 |
+
if encoding_name in ENCODINGS:
|
| 68 |
+
return ENCODINGS[encoding_name]
|
| 69 |
+
|
| 70 |
+
with _lock:
|
| 71 |
+
if encoding_name in ENCODINGS:
|
| 72 |
+
return ENCODINGS[encoding_name]
|
| 73 |
+
|
| 74 |
+
if ENCODING_CONSTRUCTORS is None:
|
| 75 |
+
_find_constructors()
|
| 76 |
+
assert ENCODING_CONSTRUCTORS is not None
|
| 77 |
+
|
| 78 |
+
if encoding_name not in ENCODING_CONSTRUCTORS:
|
| 79 |
+
raise ValueError(
|
| 80 |
+
f"Unknown encoding {encoding_name}.\n"
|
| 81 |
+
f"Plugins found: {_available_plugin_modules()}\n"
|
| 82 |
+
f"tiktoken version: {tiktoken.__version__} (are you on latest?)"
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
constructor = ENCODING_CONSTRUCTORS[encoding_name]
|
| 86 |
+
enc = Encoding(**constructor())
|
| 87 |
+
ENCODINGS[encoding_name] = enc
|
| 88 |
+
return enc
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def list_encoding_names() -> list[str]:
|
| 92 |
+
with _lock:
|
| 93 |
+
if ENCODING_CONSTRUCTORS is None:
|
| 94 |
+
_find_constructors()
|
| 95 |
+
assert ENCODING_CONSTRUCTORS is not None
|
| 96 |
+
return list(ENCODING_CONSTRUCTORS)
|