techprotrade commited on
Commit
7112995
·
verified ·
1 Parent(s): bc7505c

Add loan-advisor-agent

Browse files
.gitattributes CHANGED
@@ -708,3 +708,4 @@ loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_random.cp312-win_amd64
708
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_seq_dataset.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
709
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/sparsefuncs_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
710
  loan-advisor-agent/.venv/Lib/site-packages/sqlalchemy/cyextension/collections.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
 
 
708
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_seq_dataset.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
709
  loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/sparsefuncs_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
710
  loan-advisor-agent/.venv/Lib/site-packages/sqlalchemy/cyextension/collections.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
711
+ loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/__init__.py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # This is the public API of tiktoken
2
+ from .core import Encoding as Encoding
3
+ from .model import encoding_for_model as encoding_for_model
4
+ from .model import encoding_name_for_model as encoding_name_for_model
5
+ from .registry import get_encoding as get_encoding
6
+ from .registry import list_encoding_names as list_encoding_names
7
+
8
+ __version__ = "0.12.0"
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_educational.py ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """This is an educational implementation of the byte pair encoding algorithm."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import collections
6
+
7
+ import regex
8
+
9
+ import tiktoken
10
+
11
+
12
+ class SimpleBytePairEncoding:
13
+ def __init__(self, *, pat_str: str, mergeable_ranks: dict[bytes, int]) -> None:
14
+ """Creates an Encoding object."""
15
+ # A regex pattern string that is used to split the input text
16
+ self.pat_str = pat_str
17
+ # A dictionary mapping token bytes to their ranks. The ranks correspond to merge priority
18
+ self.mergeable_ranks = mergeable_ranks
19
+
20
+ self._decoder = {token: token_bytes for token_bytes, token in mergeable_ranks.items()}
21
+ self._pat = regex.compile(pat_str)
22
+
23
+ def encode(self, text: str, visualise: str | None = "colour") -> list[int]:
24
+ """Encodes a string into tokens.
25
+
26
+ >>> enc.encode("hello world")
27
+ [388, 372]
28
+ """
29
+ # Use the regex to split the text into (approximately) words
30
+ words = self._pat.findall(text)
31
+ tokens = []
32
+ for word in words:
33
+ # Turn each word into tokens, using the byte pair encoding algorithm
34
+ word_bytes = word.encode("utf-8")
35
+ word_tokens = bpe_encode(self.mergeable_ranks, word_bytes, visualise=visualise)
36
+ tokens.extend(word_tokens)
37
+ return tokens
38
+
39
+ def decode_bytes(self, tokens: list[int]) -> bytes:
40
+ """Decodes a list of tokens into bytes.
41
+
42
+ >>> enc.decode_bytes([388, 372])
43
+ b'hello world'
44
+ """
45
+ return b"".join(self._decoder[token] for token in tokens)
46
+
47
+ def decode(self, tokens: list[int]) -> str:
48
+ """Decodes a list of tokens into a string.
49
+
50
+ Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
51
+ the invalid bytes with the replacement character "�".
52
+
53
+ >>> enc.decode([388, 372])
54
+ 'hello world'
55
+ """
56
+ return self.decode_bytes(tokens).decode("utf-8", errors="replace")
57
+
58
+ def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]:
59
+ """Decodes a list of tokens into a list of bytes.
60
+
61
+ Useful for visualising how a string is tokenised.
62
+
63
+ >>> enc.decode_tokens_bytes([388, 372])
64
+ [b'hello', b' world']
65
+ """
66
+ return [self._decoder[token] for token in tokens]
67
+
68
+ @staticmethod
69
+ def train(training_data: str, vocab_size: int, pat_str: str):
70
+ """Train a BPE tokeniser on some data!"""
71
+ mergeable_ranks = bpe_train(data=training_data, vocab_size=vocab_size, pat_str=pat_str)
72
+ return SimpleBytePairEncoding(pat_str=pat_str, mergeable_ranks=mergeable_ranks)
73
+
74
+ @staticmethod
75
+ def from_tiktoken(encoding):
76
+ if isinstance(encoding, str):
77
+ encoding = tiktoken.get_encoding(encoding)
78
+ return SimpleBytePairEncoding(
79
+ pat_str=encoding._pat_str, mergeable_ranks=encoding._mergeable_ranks
80
+ )
81
+
82
+
83
+ def bpe_encode(
84
+ mergeable_ranks: dict[bytes, int], input: bytes, visualise: str | None = "colour"
85
+ ) -> list[int]:
86
+ parts = [bytes([b]) for b in input]
87
+ while True:
88
+ # See the intermediate merges play out!
89
+ if visualise:
90
+ if visualise in ["colour", "color"]:
91
+ visualise_tokens(parts)
92
+ elif visualise == "simple":
93
+ print(parts)
94
+
95
+ # Iterate over all pairs and find the pair we want to merge the most
96
+ min_idx = None
97
+ min_rank = None
98
+ for i, pair in enumerate(zip(parts[:-1], parts[1:])):
99
+ rank = mergeable_ranks.get(pair[0] + pair[1])
100
+ if rank is not None and (min_rank is None or rank < min_rank):
101
+ min_idx = i
102
+ min_rank = rank
103
+
104
+ # If there were no pairs we could merge, we're done!
105
+ if min_rank is None:
106
+ break
107
+ assert min_idx is not None
108
+
109
+ # Otherwise, merge that pair and leave the rest unchanged. Then repeat.
110
+ parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]
111
+
112
+ if visualise:
113
+ print()
114
+
115
+ tokens = [mergeable_ranks[part] for part in parts]
116
+ return tokens
117
+
118
+
119
+ def bpe_train(
120
+ data: str, vocab_size: int, pat_str: str, visualise: str | None = "colour"
121
+ ) -> dict[bytes, int]:
122
+ # First, add tokens for each individual byte value
123
+ if vocab_size < 2**8:
124
+ raise ValueError("vocab_size must be at least 256, so we can encode all bytes")
125
+ ranks = {}
126
+ for i in range(2**8):
127
+ ranks[bytes([i])] = i
128
+
129
+ # Splinter up our data into lists of bytes
130
+ # data = "Hello world"
131
+ # words = [
132
+ # [b'H', b'e', b'l', b'l', b'o'],
133
+ # [b' ', b'w', b'o', b'r', b'l', b'd']
134
+ # ]
135
+ words: list[list[bytes]] = [
136
+ [bytes([b]) for b in word.encode("utf-8")] for word in regex.findall(pat_str, data)
137
+ ]
138
+
139
+ # Now, use our data to figure out which merges we should make
140
+ while len(ranks) < vocab_size:
141
+ # Find the most common pair. This will become our next token
142
+ stats = collections.Counter()
143
+ for piece in words:
144
+ for pair in zip(piece[:-1], piece[1:]):
145
+ stats[pair] += 1
146
+
147
+ most_common_pair = max(stats, key=lambda x: stats[x])
148
+ token_bytes = most_common_pair[0] + most_common_pair[1]
149
+ token = len(ranks)
150
+ # Add the new token!
151
+ ranks[token_bytes] = token
152
+
153
+ # Now merge that most common pair in all the words. That is, update our training data
154
+ # to reflect our decision to make that pair into a new token.
155
+ new_words = []
156
+ for word in words:
157
+ new_word = []
158
+ i = 0
159
+ while i < len(word) - 1:
160
+ if (word[i], word[i + 1]) == most_common_pair:
161
+ # We found our pair! Merge it
162
+ new_word.append(token_bytes)
163
+ i += 2
164
+ else:
165
+ new_word.append(word[i])
166
+ i += 1
167
+ if i == len(word) - 1:
168
+ new_word.append(word[i])
169
+ new_words.append(new_word)
170
+ words = new_words
171
+
172
+ # See the intermediate merges play out!
173
+ if visualise:
174
+ print(f"The current most common pair is {most_common_pair[0]} + {most_common_pair[1]}")
175
+ print(f"So we made {token_bytes} our {len(ranks)}th token")
176
+ if visualise in ["colour", "color"]:
177
+ print("Now the first fifty words in our training data look like:")
178
+ visualise_tokens([token for word in words[:50] for token in word])
179
+ elif visualise == "simple":
180
+ print("Now the first twenty words in our training data look like:")
181
+ for word in words[:20]:
182
+ print(word)
183
+ print("\n")
184
+
185
+ return ranks
186
+
187
+
188
+ def visualise_tokens(token_values: list[bytes]) -> None:
189
+ background = [f"\u001b[48;5;{i}m" for i in [167, 179, 185, 77, 80, 68, 134]]
190
+ # If token boundaries do not occur at unicode character boundaries, it's unclear how best to
191
+ # visualise the token. Here, we'll just use the unicode replacement character to represent some
192
+ # fraction of a character.
193
+ unicode_token_values = [x.decode("utf-8", errors="replace") for x in token_values]
194
+
195
+ running_length = 0
196
+ last_color = None
197
+ for token in unicode_token_values:
198
+ color = background[running_length % len(background)]
199
+ if color == last_color:
200
+ color = background[(running_length + 1) % len(background)]
201
+ assert color != last_color
202
+ last_color = color
203
+ running_length += len(token)
204
+ print(color + token, end="")
205
+ print("\u001b[0m")
206
+
207
+
208
+ def train_simple_encoding():
209
+ gpt2_pattern = (
210
+ r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
211
+ )
212
+ with open(__file__) as f:
213
+ data = f.read()
214
+
215
+ enc = SimpleBytePairEncoding.train(data, vocab_size=600, pat_str=gpt2_pattern)
216
+
217
+ print("This is the sequence of merges performed in order to encode 'hello world':")
218
+ tokens = enc.encode("hello world")
219
+ assert enc.decode(tokens) == "hello world"
220
+ assert enc.decode_bytes(tokens) == b"hello world"
221
+ assert enc.decode_tokens_bytes(tokens) == [b"hello", b" world"]
222
+
223
+ return enc
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a3bc5da9b8838f77f7f1441e310fb6734f29d294111de25911271f3709e298e
3
+ size 2307584
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/core.py ADDED
@@ -0,0 +1,446 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import functools
4
+ from concurrent.futures import ThreadPoolExecutor
5
+ from typing import TYPE_CHECKING, AbstractSet, Collection, Literal, NoReturn, Sequence
6
+
7
+ from tiktoken import _tiktoken
8
+
9
+ if TYPE_CHECKING:
10
+ import re
11
+
12
+ import numpy as np
13
+ import numpy.typing as npt
14
+
15
+
16
+ class Encoding:
17
+ def __init__(
18
+ self,
19
+ name: str,
20
+ *,
21
+ pat_str: str,
22
+ mergeable_ranks: dict[bytes, int],
23
+ special_tokens: dict[str, int],
24
+ explicit_n_vocab: int | None = None,
25
+ ):
26
+ """Creates an Encoding object.
27
+
28
+ See openai_public.py for examples of how to construct an Encoding object.
29
+
30
+ Args:
31
+ name: The name of the encoding. It should be clear from the name of the encoding
32
+ what behaviour to expect, in particular, encodings with different special tokens
33
+ should have different names.
34
+ pat_str: A regex pattern string that is used to split the input text.
35
+ mergeable_ranks: A dictionary mapping mergeable token bytes to their ranks. The ranks
36
+ must correspond to merge priority.
37
+ special_tokens: A dictionary mapping special token strings to their token values.
38
+ explicit_n_vocab: The number of tokens in the vocabulary. If provided, it is checked
39
+ that the number of mergeable tokens and special tokens is equal to this number.
40
+ """
41
+ self.name = name
42
+
43
+ self._pat_str = pat_str
44
+ self._mergeable_ranks = mergeable_ranks
45
+ self._special_tokens = special_tokens
46
+
47
+ self.max_token_value = max(
48
+ max(mergeable_ranks.values()), max(special_tokens.values(), default=0)
49
+ )
50
+ if explicit_n_vocab:
51
+ assert len(mergeable_ranks) + len(special_tokens) == explicit_n_vocab
52
+ assert self.max_token_value == explicit_n_vocab - 1
53
+
54
+ self._core_bpe = _tiktoken.CoreBPE(mergeable_ranks, special_tokens, pat_str)
55
+
56
+ def __repr__(self) -> str:
57
+ return f"<Encoding {self.name!r}>"
58
+
59
+ # ====================
60
+ # Encoding
61
+ # ====================
62
+
63
+ def encode_ordinary(self, text: str) -> list[int]:
64
+ """Encodes a string into tokens, ignoring special tokens.
65
+
66
+ This is equivalent to `encode(text, disallowed_special=())` (but slightly faster).
67
+
68
+ ```
69
+ >>> enc.encode_ordinary("hello world")
70
+ [31373, 995]
71
+ """
72
+ try:
73
+ return self._core_bpe.encode_ordinary(text)
74
+ except UnicodeEncodeError:
75
+ # See comment in encode
76
+ text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
77
+ return self._core_bpe.encode_ordinary(text)
78
+
79
+ def encode(
80
+ self,
81
+ text: str,
82
+ *,
83
+ allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
84
+ disallowed_special: Literal["all"] | Collection[str] = "all",
85
+ ) -> list[int]:
86
+ """Encodes a string into tokens.
87
+
88
+ Special tokens are artificial tokens used to unlock capabilities from a model,
89
+ such as fill-in-the-middle. So we want to be careful about accidentally encoding special
90
+ tokens, since they can be used to trick a model into doing something we don't want it to do.
91
+
92
+ Hence, by default, encode will raise an error if it encounters text that corresponds
93
+ to a special token. This can be controlled on a per-token level using the `allowed_special`
94
+ and `disallowed_special` parameters. In particular:
95
+ - Setting `disallowed_special` to () will prevent this function from raising errors and
96
+ cause all text corresponding to special tokens to be encoded as natural text.
97
+ - Setting `allowed_special` to "all" will cause this function to treat all text
98
+ corresponding to special tokens to be encoded as special tokens.
99
+
100
+ ```
101
+ >>> enc.encode("hello world")
102
+ [31373, 995]
103
+ >>> enc.encode("<|endoftext|>", allowed_special={"<|endoftext|>"})
104
+ [50256]
105
+ >>> enc.encode("<|endoftext|>", allowed_special="all")
106
+ [50256]
107
+ >>> enc.encode("<|endoftext|>")
108
+ # Raises ValueError
109
+ >>> enc.encode("<|endoftext|>", disallowed_special=())
110
+ [27, 91, 437, 1659, 5239, 91, 29]
111
+ ```
112
+ """
113
+ if allowed_special == "all":
114
+ allowed_special = self.special_tokens_set
115
+ if disallowed_special == "all":
116
+ disallowed_special = self.special_tokens_set - allowed_special
117
+ if disallowed_special:
118
+ if not isinstance(disallowed_special, frozenset):
119
+ disallowed_special = frozenset(disallowed_special)
120
+ if match := _special_token_regex(disallowed_special).search(text):
121
+ raise_disallowed_special_token(match.group())
122
+
123
+ try:
124
+ return self._core_bpe.encode(text, allowed_special)
125
+ except UnicodeEncodeError:
126
+ # BPE operates on bytes, but the regex operates on unicode. If we pass a str that is
127
+ # invalid UTF-8 to Rust, it will rightfully complain. Here we do a quick and dirty
128
+ # fixup for any surrogate pairs that may have sneaked their way into the text.
129
+ # Technically, this introduces a place where encode + decode doesn't roundtrip a Python
130
+ # string, but given that this is input we want to support, maybe that's okay.
131
+ # Also we use errors="replace" to handle weird things like lone surrogates.
132
+ text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
133
+ return self._core_bpe.encode(text, allowed_special)
134
+
135
+ def encode_to_numpy(
136
+ self,
137
+ text: str,
138
+ *,
139
+ allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
140
+ disallowed_special: Literal["all"] | Collection[str] = "all",
141
+ ) -> npt.NDArray[np.uint32]:
142
+ """Encodes a string into tokens, returning a numpy array.
143
+
144
+ Avoids the overhead of copying the token buffer into a Python list.
145
+ """
146
+ if allowed_special == "all":
147
+ allowed_special = self.special_tokens_set
148
+ if disallowed_special == "all":
149
+ disallowed_special = self.special_tokens_set - allowed_special
150
+ if disallowed_special:
151
+ if not isinstance(disallowed_special, frozenset):
152
+ disallowed_special = frozenset(disallowed_special)
153
+ if match := _special_token_regex(disallowed_special).search(text):
154
+ raise_disallowed_special_token(match.group())
155
+
156
+ import numpy as np
157
+
158
+ buffer = self._core_bpe.encode_to_tiktoken_buffer(text, allowed_special)
159
+ return np.frombuffer(buffer, dtype=np.uint32)
160
+
161
+ def encode_ordinary_batch(self, text: list[str], *, num_threads: int = 8) -> list[list[int]]:
162
+ """Encodes a list of strings into tokens, in parallel, ignoring special tokens.
163
+
164
+ This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster).
165
+
166
+ ```
167
+ >>> enc.encode_ordinary_batch(["hello world", "goodbye world"])
168
+ [[31373, 995], [11274, 16390, 995]]
169
+ ```
170
+ """
171
+ encoder = functools.partial(self.encode_ordinary)
172
+ with ThreadPoolExecutor(num_threads) as e:
173
+ return list(e.map(encoder, text))
174
+
175
+ def encode_batch(
176
+ self,
177
+ text: list[str],
178
+ *,
179
+ num_threads: int = 8,
180
+ allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
181
+ disallowed_special: Literal["all"] | Collection[str] = "all",
182
+ ) -> list[list[int]]:
183
+ """Encodes a list of strings into tokens, in parallel.
184
+
185
+ See `encode` for more details on `allowed_special` and `disallowed_special`.
186
+
187
+ ```
188
+ >>> enc.encode_batch(["hello world", "goodbye world"])
189
+ [[31373, 995], [11274, 16390, 995]]
190
+ ```
191
+ """
192
+ if allowed_special == "all":
193
+ allowed_special = self.special_tokens_set
194
+ if disallowed_special == "all":
195
+ disallowed_special = self.special_tokens_set - allowed_special
196
+ if not isinstance(disallowed_special, frozenset):
197
+ disallowed_special = frozenset(disallowed_special)
198
+
199
+ encoder = functools.partial(
200
+ self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special
201
+ )
202
+ with ThreadPoolExecutor(num_threads) as e:
203
+ return list(e.map(encoder, text))
204
+
205
+ def encode_with_unstable(
206
+ self,
207
+ text: str,
208
+ *,
209
+ allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006
210
+ disallowed_special: Literal["all"] | Collection[str] = "all",
211
+ ) -> tuple[list[int], list[list[int]]]:
212
+ """Encodes a string into stable tokens and possible completion sequences.
213
+
214
+ Note that the stable tokens will only represent a substring of `text`.
215
+
216
+ See `encode` for more details on `allowed_special` and `disallowed_special`.
217
+
218
+ This API should itself be considered unstable.
219
+
220
+ ```
221
+ >>> enc.encode_with_unstable("hello fanta")
222
+ ([31373], [(277, 4910), (5113, 265), ..., (8842,)])
223
+
224
+ >>> text = "..."
225
+ >>> stable_tokens, completions = enc.encode_with_unstable(text)
226
+ >>> assert text.encode().startswith(enc.decode_bytes(stable_tokens))
227
+ >>> assert all(enc.decode_bytes(stable_tokens + seq).startswith(text.encode()) for seq in completions)
228
+ ```
229
+ """
230
+ if allowed_special == "all":
231
+ allowed_special = self.special_tokens_set
232
+ if disallowed_special == "all":
233
+ disallowed_special = self.special_tokens_set - allowed_special
234
+ if disallowed_special:
235
+ if not isinstance(disallowed_special, frozenset):
236
+ disallowed_special = frozenset(disallowed_special)
237
+ if match := _special_token_regex(disallowed_special).search(text):
238
+ raise_disallowed_special_token(match.group())
239
+
240
+ return self._core_bpe.encode_with_unstable(text, allowed_special)
241
+
242
+ def encode_single_token(self, text_or_bytes: str | bytes) -> int:
243
+ """Encodes text corresponding to a single token to its token value.
244
+
245
+ NOTE: this will encode all special tokens.
246
+
247
+ Raises `KeyError` if the token is not in the vocabulary.
248
+
249
+ ```
250
+ >>> enc.encode_single_token("hello")
251
+ 31373
252
+ ```
253
+ """
254
+ if isinstance(text_or_bytes, str):
255
+ text_or_bytes = text_or_bytes.encode("utf-8")
256
+ return self._core_bpe.encode_single_token(text_or_bytes)
257
+
258
+ # ====================
259
+ # Decoding
260
+ # ====================
261
+
262
+ def decode_bytes(self, tokens: Sequence[int]) -> bytes:
263
+ """Decodes a list of tokens into bytes.
264
+
265
+ ```
266
+ >>> enc.decode_bytes([31373, 995])
267
+ b'hello world'
268
+ ```
269
+ """
270
+ return self._core_bpe.decode_bytes(tokens)
271
+
272
+ def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
273
+ """Decodes a list of tokens into a string.
274
+
275
+ WARNING: the default behaviour of this function is lossy, since decoded bytes are not
276
+ guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter,
277
+ for instance, setting `errors=strict`.
278
+
279
+ ```
280
+ >>> enc.decode([31373, 995])
281
+ 'hello world'
282
+ ```
283
+ """
284
+ return self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors)
285
+
286
+ def decode_single_token_bytes(self, token: int) -> bytes:
287
+ """Decodes a token into bytes.
288
+
289
+ NOTE: this will decode all special tokens.
290
+
291
+ Raises `KeyError` if the token is not in the vocabulary.
292
+
293
+ ```
294
+ >>> enc.decode_single_token_bytes(31373)
295
+ b'hello'
296
+ ```
297
+ """
298
+ return self._core_bpe.decode_single_token_bytes(token)
299
+
300
+ def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]:
301
+ """Decodes a list of tokens into a list of bytes.
302
+
303
+ Useful for visualising tokenisation.
304
+ >>> enc.decode_tokens_bytes([31373, 995])
305
+ [b'hello', b' world']
306
+ """
307
+ return [self.decode_single_token_bytes(token) for token in tokens]
308
+
309
+ def decode_with_offsets(self, tokens: Sequence[int]) -> tuple[str, list[int]]:
310
+ """Decodes a list of tokens into a string and a list of offsets.
311
+
312
+ Each offset is the index into text corresponding to the start of each token.
313
+ If UTF-8 character boundaries do not line up with token boundaries, the offset is the index
314
+ of the first character that contains bytes from the token.
315
+
316
+ This will currently raise if given tokens that decode to invalid UTF-8; this behaviour may
317
+ change in the future to be more permissive.
318
+
319
+ >>> enc.decode_with_offsets([31373, 995])
320
+ ('hello world', [0, 5])
321
+ """
322
+ token_bytes = self.decode_tokens_bytes(tokens)
323
+
324
+ text_len = 0
325
+ offsets = []
326
+ for token in token_bytes:
327
+ offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0)))
328
+ text_len += sum(1 for c in token if not 0x80 <= c < 0xC0)
329
+
330
+ # TODO: assess correctness for errors="ignore" and errors="replace"
331
+ text = b"".join(token_bytes).decode("utf-8", errors="strict")
332
+ return text, offsets
333
+
334
+ def decode_batch(
335
+ self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
336
+ ) -> list[str]:
337
+ """Decodes a batch (list of lists of tokens) into a list of strings."""
338
+ decoder = functools.partial(self.decode, errors=errors)
339
+ with ThreadPoolExecutor(num_threads) as e:
340
+ return list(e.map(decoder, batch))
341
+
342
+ def decode_bytes_batch(
343
+ self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
344
+ ) -> list[bytes]:
345
+ """Decodes a batch (list of lists of tokens) into a list of bytes."""
346
+ with ThreadPoolExecutor(num_threads) as e:
347
+ return list(e.map(self.decode_bytes, batch))
348
+
349
+ # ====================
350
+ # Miscellaneous
351
+ # ====================
352
+
353
+ def token_byte_values(self) -> list[bytes]:
354
+ """Returns the list of all token byte values."""
355
+ return self._core_bpe.token_byte_values()
356
+
357
+ @property
358
+ def eot_token(self) -> int:
359
+ return self._special_tokens["<|endoftext|>"]
360
+
361
+ @functools.cached_property
362
+ def special_tokens_set(self) -> set[str]:
363
+ return set(self._special_tokens.keys())
364
+
365
+ def is_special_token(self, token: int) -> bool:
366
+ assert isinstance(token, int)
367
+ return token in self._special_token_values
368
+
369
+ @property
370
+ def n_vocab(self) -> int:
371
+ """For backwards compatibility. Prefer to use `enc.max_token_value + 1`."""
372
+ return self.max_token_value + 1
373
+
374
+ # ====================
375
+ # Private
376
+ # ====================
377
+
378
+ def _encode_single_piece(self, text_or_bytes: str | bytes) -> list[int]:
379
+ """Encodes text corresponding to bytes without a regex split.
380
+
381
+ NOTE: this will not encode any special tokens.
382
+
383
+ ```
384
+ >>> enc.encode_single_piece("helloqqqq")
385
+ [31373, 38227, 38227]
386
+ ```
387
+ """
388
+ if isinstance(text_or_bytes, str):
389
+ text_or_bytes = text_or_bytes.encode("utf-8")
390
+ return self._core_bpe.encode_single_piece(text_or_bytes)
391
+
392
+ def _encode_only_native_bpe(self, text: str) -> list[int]:
393
+ """Encodes a string into tokens, but do regex splitting in Python."""
394
+ # We need specifically `regex` in order to compile pat_str due to e.g. \p
395
+ import regex
396
+
397
+ _unused_pat = regex.compile(self._pat_str)
398
+ ret = []
399
+ for piece in regex.findall(_unused_pat, text):
400
+ ret.extend(self._core_bpe.encode_single_piece(piece.encode("utf-8")))
401
+ return ret
402
+
403
+ def _encode_bytes(self, text: bytes) -> list[int]:
404
+ return self._core_bpe._encode_bytes(text)
405
+
406
+ def __getstate__(self) -> object:
407
+ import tiktoken.registry
408
+
409
+ # As an optimisation, pickle registered encodings by reference
410
+ if self is tiktoken.registry.ENCODINGS.get(self.name):
411
+ return self.name
412
+ return {
413
+ "name": self.name,
414
+ "pat_str": self._pat_str,
415
+ "mergeable_ranks": self._mergeable_ranks,
416
+ "special_tokens": self._special_tokens,
417
+ }
418
+
419
+ def __setstate__(self, value: object) -> None:
420
+ import tiktoken.registry
421
+
422
+ if isinstance(value, str):
423
+ self.__dict__ = tiktoken.registry.get_encoding(value).__dict__
424
+ return
425
+ self.__init__(**value)
426
+
427
+
428
+ @functools.lru_cache(maxsize=128)
429
+ def _special_token_regex(tokens: frozenset[str]) -> re.Pattern[str]:
430
+ try:
431
+ import regex as re
432
+ except ImportError:
433
+ import re
434
+ inner = "|".join(re.escape(token) for token in tokens)
435
+ return re.compile(f"({inner})")
436
+
437
+
438
+ def raise_disallowed_special_token(token: str) -> NoReturn:
439
+ raise ValueError(
440
+ f"Encountered text corresponding to disallowed special token {token!r}.\n"
441
+ "If you want this text to be encoded as a special token, "
442
+ f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"
443
+ f"If you want this text to be encoded as normal text, disable the check for this token "
444
+ f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n"
445
+ "To disable this check for all special tokens, pass `disallowed_special=()`.\n"
446
+ )
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/load.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import hashlib
5
+ import os
6
+
7
+
8
+ def read_file(blobpath: str) -> bytes:
9
+ if "://" not in blobpath:
10
+ with open(blobpath, "rb", buffering=0) as f:
11
+ return f.read()
12
+
13
+ if blobpath.startswith(("http://", "https://")):
14
+ # avoiding blobfile for public files helps avoid auth issues, like MFA prompts.
15
+ import requests
16
+
17
+ resp = requests.get(blobpath)
18
+ resp.raise_for_status()
19
+ return resp.content
20
+
21
+ try:
22
+ import blobfile
23
+ except ImportError as e:
24
+ raise ImportError(
25
+ "blobfile is not installed. Please install it by running `pip install blobfile`."
26
+ ) from e
27
+ with blobfile.BlobFile(blobpath, "rb") as f:
28
+ return f.read()
29
+
30
+
31
+ def check_hash(data: bytes, expected_hash: str) -> bool:
32
+ actual_hash = hashlib.sha256(data).hexdigest()
33
+ return actual_hash == expected_hash
34
+
35
+
36
+ def read_file_cached(blobpath: str, expected_hash: str | None = None) -> bytes:
37
+ user_specified_cache = True
38
+ if "TIKTOKEN_CACHE_DIR" in os.environ:
39
+ cache_dir = os.environ["TIKTOKEN_CACHE_DIR"]
40
+ elif "DATA_GYM_CACHE_DIR" in os.environ:
41
+ cache_dir = os.environ["DATA_GYM_CACHE_DIR"]
42
+ else:
43
+ import tempfile
44
+
45
+ cache_dir = os.path.join(tempfile.gettempdir(), "data-gym-cache")
46
+ user_specified_cache = False
47
+
48
+ if cache_dir == "":
49
+ # disable caching
50
+ return read_file(blobpath)
51
+
52
+ cache_key = hashlib.sha1(blobpath.encode()).hexdigest()
53
+
54
+ cache_path = os.path.join(cache_dir, cache_key)
55
+ if os.path.exists(cache_path):
56
+ with open(cache_path, "rb", buffering=0) as f:
57
+ data = f.read()
58
+ if expected_hash is None or check_hash(data, expected_hash):
59
+ return data
60
+
61
+ # the cached file does not match the hash, remove it and re-fetch
62
+ try:
63
+ os.remove(cache_path)
64
+ except OSError:
65
+ pass
66
+
67
+ contents = read_file(blobpath)
68
+ if expected_hash and not check_hash(contents, expected_hash):
69
+ raise ValueError(
70
+ f"Hash mismatch for data downloaded from {blobpath} (expected {expected_hash}). "
71
+ f"This may indicate a corrupted download. Please try again."
72
+ )
73
+
74
+ import uuid
75
+
76
+ try:
77
+ os.makedirs(cache_dir, exist_ok=True)
78
+ tmp_filename = cache_path + "." + str(uuid.uuid4()) + ".tmp"
79
+ with open(tmp_filename, "wb") as f:
80
+ f.write(contents)
81
+ os.rename(tmp_filename, cache_path)
82
+ except OSError:
83
+ # don't raise if we can't write to the default cache, e.g. issue #75
84
+ if user_specified_cache:
85
+ raise
86
+
87
+ return contents
88
+
89
+
90
+ def data_gym_to_mergeable_bpe_ranks(
91
+ vocab_bpe_file: str,
92
+ encoder_json_file: str,
93
+ vocab_bpe_hash: str | None = None,
94
+ encoder_json_hash: str | None = None,
95
+ clobber_one_byte_tokens: bool = False,
96
+ ) -> dict[bytes, int]:
97
+ # NB: do not add caching to this function
98
+ rank_to_intbyte = [b for b in range(2**8) if chr(b).isprintable() and chr(b) != " "]
99
+
100
+ data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte}
101
+ n = 0
102
+ for b in range(2**8):
103
+ if b not in rank_to_intbyte:
104
+ rank_to_intbyte.append(b)
105
+ data_gym_byte_to_byte[chr(2**8 + n)] = b
106
+ n += 1
107
+ assert len(rank_to_intbyte) == 2**8
108
+
109
+ # vocab_bpe contains the merges along with associated ranks
110
+ vocab_bpe_contents = read_file_cached(vocab_bpe_file, vocab_bpe_hash).decode()
111
+ bpe_merges = [tuple(merge_str.split()) for merge_str in vocab_bpe_contents.split("\n")[1:-1]]
112
+
113
+ def decode_data_gym(value: str) -> bytes:
114
+ return bytes(data_gym_byte_to_byte[b] for b in value)
115
+
116
+ # add the single byte tokens
117
+ # if clobber_one_byte_tokens is True, we'll replace these with ones from the encoder json
118
+ bpe_ranks = {bytes([b]): i for i, b in enumerate(rank_to_intbyte)}
119
+ del rank_to_intbyte
120
+
121
+ # add the merged tokens
122
+ n = len(bpe_ranks)
123
+ for first, second in bpe_merges:
124
+ bpe_ranks[decode_data_gym(first) + decode_data_gym(second)] = n
125
+ n += 1
126
+
127
+ import json
128
+
129
+ # check that the encoder file matches the merges file
130
+ # this sanity check is important since tiktoken assumes that ranks are ordered the same
131
+ # as merge priority
132
+ encoder_json = json.loads(read_file_cached(encoder_json_file, encoder_json_hash))
133
+ encoder_json_loaded = {decode_data_gym(k): v for k, v in encoder_json.items()}
134
+ # drop these two special tokens if present, since they're not mergeable bpe tokens
135
+ encoder_json_loaded.pop(b"<|endoftext|>", None)
136
+ encoder_json_loaded.pop(b"<|startoftext|>", None)
137
+
138
+ if clobber_one_byte_tokens:
139
+ for k in encoder_json_loaded:
140
+ if len(k) == 1:
141
+ bpe_ranks[k] = encoder_json_loaded[k]
142
+
143
+ assert bpe_ranks == encoder_json_loaded
144
+
145
+ return bpe_ranks
146
+
147
+
148
+ def dump_tiktoken_bpe(bpe_ranks: dict[bytes, int], tiktoken_bpe_file: str) -> None:
149
+ try:
150
+ import blobfile
151
+ except ImportError as e:
152
+ raise ImportError(
153
+ "blobfile is not installed. Please install it by running `pip install blobfile`."
154
+ ) from e
155
+ with blobfile.BlobFile(tiktoken_bpe_file, "wb") as f:
156
+ for token, rank in sorted(bpe_ranks.items(), key=lambda x: x[1]):
157
+ f.write(base64.b64encode(token) + b" " + str(rank).encode() + b"\n")
158
+
159
+
160
+ def load_tiktoken_bpe(tiktoken_bpe_file: str, expected_hash: str | None = None) -> dict[bytes, int]:
161
+ # NB: do not add caching to this function
162
+ contents = read_file_cached(tiktoken_bpe_file, expected_hash)
163
+ ret = {}
164
+ for line in contents.splitlines():
165
+ if not line:
166
+ continue
167
+ try:
168
+ token, rank = line.split()
169
+ ret[base64.b64decode(token)] = int(rank)
170
+ except Exception as e:
171
+ raise ValueError(f"Error parsing line {line!r} in {tiktoken_bpe_file}") from e
172
+ return ret
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/model.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from .core import Encoding
4
+ from .registry import get_encoding
5
+
6
+ # TODO: these will likely be replaced by an API endpoint
7
+ MODEL_PREFIX_TO_ENCODING: dict[str, str] = {
8
+ "o1-": "o200k_base",
9
+ "o3-": "o200k_base",
10
+ "o4-mini-": "o200k_base",
11
+ # chat
12
+ "gpt-5-": "o200k_base",
13
+ "gpt-4.5-": "o200k_base",
14
+ "gpt-4.1-": "o200k_base",
15
+ "chatgpt-4o-": "o200k_base",
16
+ "gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13
17
+ "gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k
18
+ "gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc.
19
+ "gpt-35-turbo-": "cl100k_base", # Azure deployment name
20
+ "gpt-oss-": "o200k_harmony",
21
+ # fine-tuned
22
+ "ft:gpt-4o": "o200k_base",
23
+ "ft:gpt-4": "cl100k_base",
24
+ "ft:gpt-3.5-turbo": "cl100k_base",
25
+ "ft:davinci-002": "cl100k_base",
26
+ "ft:babbage-002": "cl100k_base",
27
+ }
28
+
29
+ MODEL_TO_ENCODING: dict[str, str] = {
30
+ # reasoning
31
+ "o1": "o200k_base",
32
+ "o3": "o200k_base",
33
+ "o4-mini": "o200k_base",
34
+ # chat
35
+ "gpt-5": "o200k_base",
36
+ "gpt-4.1": "o200k_base",
37
+ "gpt-4o": "o200k_base",
38
+ "gpt-4": "cl100k_base",
39
+ "gpt-3.5-turbo": "cl100k_base",
40
+ "gpt-3.5": "cl100k_base", # Common shorthand
41
+ "gpt-35-turbo": "cl100k_base", # Azure deployment name
42
+ # base
43
+ "davinci-002": "cl100k_base",
44
+ "babbage-002": "cl100k_base",
45
+ # embeddings
46
+ "text-embedding-ada-002": "cl100k_base",
47
+ "text-embedding-3-small": "cl100k_base",
48
+ "text-embedding-3-large": "cl100k_base",
49
+ # DEPRECATED MODELS
50
+ # text (DEPRECATED)
51
+ "text-davinci-003": "p50k_base",
52
+ "text-davinci-002": "p50k_base",
53
+ "text-davinci-001": "r50k_base",
54
+ "text-curie-001": "r50k_base",
55
+ "text-babbage-001": "r50k_base",
56
+ "text-ada-001": "r50k_base",
57
+ "davinci": "r50k_base",
58
+ "curie": "r50k_base",
59
+ "babbage": "r50k_base",
60
+ "ada": "r50k_base",
61
+ # code (DEPRECATED)
62
+ "code-davinci-002": "p50k_base",
63
+ "code-davinci-001": "p50k_base",
64
+ "code-cushman-002": "p50k_base",
65
+ "code-cushman-001": "p50k_base",
66
+ "davinci-codex": "p50k_base",
67
+ "cushman-codex": "p50k_base",
68
+ # edit (DEPRECATED)
69
+ "text-davinci-edit-001": "p50k_edit",
70
+ "code-davinci-edit-001": "p50k_edit",
71
+ # old embeddings (DEPRECATED)
72
+ "text-similarity-davinci-001": "r50k_base",
73
+ "text-similarity-curie-001": "r50k_base",
74
+ "text-similarity-babbage-001": "r50k_base",
75
+ "text-similarity-ada-001": "r50k_base",
76
+ "text-search-davinci-doc-001": "r50k_base",
77
+ "text-search-curie-doc-001": "r50k_base",
78
+ "text-search-babbage-doc-001": "r50k_base",
79
+ "text-search-ada-doc-001": "r50k_base",
80
+ "code-search-babbage-code-001": "r50k_base",
81
+ "code-search-ada-code-001": "r50k_base",
82
+ # open source
83
+ "gpt2": "gpt2",
84
+ "gpt-2": "gpt2", # Maintains consistency with gpt-4
85
+ }
86
+
87
+
88
+ def encoding_name_for_model(model_name: str) -> str:
89
+ """Returns the name of the encoding used by a model.
90
+
91
+ Raises a KeyError if the model name is not recognised.
92
+ """
93
+ encoding_name = None
94
+ if model_name in MODEL_TO_ENCODING:
95
+ encoding_name = MODEL_TO_ENCODING[model_name]
96
+ else:
97
+ # Check if the model matches a known prefix
98
+ # Prefix matching avoids needing library updates for every model version release
99
+ # Note that this can match on non-existent models (e.g., gpt-3.5-turbo-FAKE)
100
+ for model_prefix, model_encoding_name in MODEL_PREFIX_TO_ENCODING.items():
101
+ if model_name.startswith(model_prefix):
102
+ return model_encoding_name
103
+
104
+ if encoding_name is None:
105
+ raise KeyError(
106
+ f"Could not automatically map {model_name} to a tokeniser. "
107
+ "Please use `tiktoken.get_encoding` to explicitly get the tokeniser you expect."
108
+ ) from None
109
+
110
+ return encoding_name
111
+
112
+
113
+ def encoding_for_model(model_name: str) -> Encoding:
114
+ """Returns the encoding used by a model.
115
+
116
+ Raises a KeyError if the model name is not recognised.
117
+ """
118
+ return get_encoding(encoding_name_for_model(model_name))
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/py.typed ADDED
File without changes
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/registry.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import functools
4
+ import importlib
5
+ import pkgutil
6
+ import threading
7
+ from typing import Any, Callable, Sequence
8
+
9
+ import tiktoken_ext
10
+
11
+ import tiktoken
12
+ from tiktoken.core import Encoding
13
+
14
+ _lock = threading.RLock()
15
+ ENCODINGS: dict[str, Encoding] = {}
16
+ ENCODING_CONSTRUCTORS: dict[str, Callable[[], dict[str, Any]]] | None = None
17
+
18
+
19
+ @functools.lru_cache
20
+ def _available_plugin_modules() -> Sequence[str]:
21
+ # tiktoken_ext is a namespace package
22
+ # submodules inside tiktoken_ext will be inspected for ENCODING_CONSTRUCTORS attributes
23
+ # - we use namespace package pattern so `pkgutil.iter_modules` is fast
24
+ # - it's a separate top-level package because namespace subpackages of non-namespace
25
+ # packages don't quite do what you want with editable installs
26
+ mods = []
27
+ plugin_mods = pkgutil.iter_modules(tiktoken_ext.__path__, tiktoken_ext.__name__ + ".")
28
+ for _, mod_name, _ in plugin_mods:
29
+ mods.append(mod_name)
30
+ return mods
31
+
32
+
33
+ def _find_constructors() -> None:
34
+ global ENCODING_CONSTRUCTORS
35
+ with _lock:
36
+ if ENCODING_CONSTRUCTORS is not None:
37
+ return
38
+ ENCODING_CONSTRUCTORS = {}
39
+
40
+ try:
41
+ for mod_name in _available_plugin_modules():
42
+ mod = importlib.import_module(mod_name)
43
+ try:
44
+ constructors = mod.ENCODING_CONSTRUCTORS
45
+ except AttributeError as e:
46
+ raise ValueError(
47
+ f"tiktoken plugin {mod_name} does not define ENCODING_CONSTRUCTORS"
48
+ ) from e
49
+ for enc_name, constructor in constructors.items():
50
+ if enc_name in ENCODING_CONSTRUCTORS:
51
+ raise ValueError(
52
+ f"Duplicate encoding name {enc_name} in tiktoken plugin {mod_name}"
53
+ )
54
+ ENCODING_CONSTRUCTORS[enc_name] = constructor
55
+ except Exception:
56
+ # Ensure we idempotently raise errors
57
+ ENCODING_CONSTRUCTORS = None
58
+ raise
59
+
60
+
61
+
62
+
63
+ def get_encoding(encoding_name: str) -> Encoding:
64
+ if not isinstance(encoding_name, str):
65
+ raise ValueError(f"Expected a string in get_encoding, got {type(encoding_name)}")
66
+
67
+ if encoding_name in ENCODINGS:
68
+ return ENCODINGS[encoding_name]
69
+
70
+ with _lock:
71
+ if encoding_name in ENCODINGS:
72
+ return ENCODINGS[encoding_name]
73
+
74
+ if ENCODING_CONSTRUCTORS is None:
75
+ _find_constructors()
76
+ assert ENCODING_CONSTRUCTORS is not None
77
+
78
+ if encoding_name not in ENCODING_CONSTRUCTORS:
79
+ raise ValueError(
80
+ f"Unknown encoding {encoding_name}.\n"
81
+ f"Plugins found: {_available_plugin_modules()}\n"
82
+ f"tiktoken version: {tiktoken.__version__} (are you on latest?)"
83
+ )
84
+
85
+ constructor = ENCODING_CONSTRUCTORS[encoding_name]
86
+ enc = Encoding(**constructor())
87
+ ENCODINGS[encoding_name] = enc
88
+ return enc
89
+
90
+
91
+ def list_encoding_names() -> list[str]:
92
+ with _lock:
93
+ if ENCODING_CONSTRUCTORS is None:
94
+ _find_constructors()
95
+ assert ENCODING_CONSTRUCTORS is not None
96
+ return list(ENCODING_CONSTRUCTORS)