Add loan-advisor-agent
Browse files- .gitattributes +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/__init__.py +100 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/__init__.pyi +1800 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/decoders/__init__.py +15 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/decoders/__init__.pyi +569 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/__init__.py +6 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/base_tokenizer.py +459 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/bert_wordpiece.py +151 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/byte_level_bpe.py +122 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/char_level_bpe.py +150 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_bpe.py +103 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_unigram.py +196 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/models/__init__.py +8 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/models/__init__.pyi +744 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/normalizers/__init__.py +29 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/normalizers/__init__.pyi +946 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.py +16 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.pyi +1015 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/processors/__init__.py +9 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/processors/__init__.pyi +519 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tokenizers.pyd +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tokenizers.pyi +17 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tools/__init__.py +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tools/visualizer-styles.css +170 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tools/visualizer.py +407 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/trainers/__init__.py +8 -0
- loan-advisor-agent/.venv/Lib/site-packages/tokenizers/trainers/__init__.pyi +462 -0
.gitattributes
CHANGED
|
@@ -709,3 +709,4 @@ loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/_seq_dataset.cp312-win_
|
|
| 709 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/sparsefuncs_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 710 |
loan-advisor-agent/.venv/Lib/site-packages/sqlalchemy/cyextension/collections.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 711 |
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 709 |
loan-advisor-agent/.venv/Lib/site-packages/sklearn/utils/sparsefuncs_fast.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 710 |
loan-advisor-agent/.venv/Lib/site-packages/sqlalchemy/cyextension/collections.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 711 |
loan-advisor-agent/.venv/Lib/site-packages/tiktoken/_tiktoken.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 712 |
+
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tokenizers.pyd filter=lfs diff=lfs merge=lfs -text
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/__init__.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from enum import Enum
|
| 2 |
+
from typing import List, Tuple, Union
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
Offsets = Tuple[int, int]
|
| 6 |
+
|
| 7 |
+
TextInputSequence = str
|
| 8 |
+
"""A :obj:`str` that represents an input sequence """
|
| 9 |
+
|
| 10 |
+
PreTokenizedInputSequence = Union[List[str], Tuple[str]]
|
| 11 |
+
"""A pre-tokenized input sequence. Can be one of:
|
| 12 |
+
|
| 13 |
+
- A :obj:`List` of :obj:`str`
|
| 14 |
+
- A :obj:`Tuple` of :obj:`str`
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
TextEncodeInput = Union[
|
| 18 |
+
TextInputSequence,
|
| 19 |
+
Tuple[TextInputSequence, TextInputSequence],
|
| 20 |
+
List[TextInputSequence],
|
| 21 |
+
]
|
| 22 |
+
"""Represents a textual input for encoding. Can be either:
|
| 23 |
+
|
| 24 |
+
- A single sequence: :data:`~tokenizers.TextInputSequence`
|
| 25 |
+
- A pair of sequences:
|
| 26 |
+
|
| 27 |
+
- A :obj:`Tuple` of :data:`~tokenizers.TextInputSequence`
|
| 28 |
+
- Or a :obj:`List` of :data:`~tokenizers.TextInputSequence` of size 2
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
PreTokenizedEncodeInput = Union[
|
| 32 |
+
PreTokenizedInputSequence,
|
| 33 |
+
Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
|
| 34 |
+
List[PreTokenizedInputSequence],
|
| 35 |
+
]
|
| 36 |
+
"""Represents a pre-tokenized input for encoding. Can be either:
|
| 37 |
+
|
| 38 |
+
- A single sequence: :data:`~tokenizers.PreTokenizedInputSequence`
|
| 39 |
+
- A pair of sequences:
|
| 40 |
+
|
| 41 |
+
- A :obj:`Tuple` of :data:`~tokenizers.PreTokenizedInputSequence`
|
| 42 |
+
- Or a :obj:`List` of :data:`~tokenizers.PreTokenizedInputSequence` of size 2
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
|
| 46 |
+
"""Represents all the possible types of input sequences for encoding. Can be:
|
| 47 |
+
|
| 48 |
+
- When ``is_pretokenized=False``: :data:`~TextInputSequence`
|
| 49 |
+
- When ``is_pretokenized=True``: :data:`~PreTokenizedInputSequence`
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
|
| 53 |
+
"""Represents all the possible types of input for encoding. Can be:
|
| 54 |
+
|
| 55 |
+
- When ``is_pretokenized=False``: :data:`~TextEncodeInput`
|
| 56 |
+
- When ``is_pretokenized=True``: :data:`~PreTokenizedEncodeInput`
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class OffsetReferential(Enum):
|
| 61 |
+
ORIGINAL = "original"
|
| 62 |
+
NORMALIZED = "normalized"
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
class OffsetType(Enum):
|
| 66 |
+
BYTE = "byte"
|
| 67 |
+
CHAR = "char"
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class SplitDelimiterBehavior(Enum):
|
| 71 |
+
REMOVED = "removed"
|
| 72 |
+
ISOLATED = "isolated"
|
| 73 |
+
MERGED_WITH_PREVIOUS = "merged_with_previous"
|
| 74 |
+
MERGED_WITH_NEXT = "merged_with_next"
|
| 75 |
+
CONTIGUOUS = "contiguous"
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
from .tokenizers import ( # type: ignore[import]
|
| 79 |
+
AddedToken,
|
| 80 |
+
Encoding,
|
| 81 |
+
NormalizedString,
|
| 82 |
+
PreTokenizedString,
|
| 83 |
+
Regex,
|
| 84 |
+
Token,
|
| 85 |
+
Tokenizer,
|
| 86 |
+
decoders,
|
| 87 |
+
models,
|
| 88 |
+
normalizers,
|
| 89 |
+
pre_tokenizers,
|
| 90 |
+
processors,
|
| 91 |
+
trainers,
|
| 92 |
+
__version__,
|
| 93 |
+
)
|
| 94 |
+
from .implementations import (
|
| 95 |
+
BertWordPieceTokenizer,
|
| 96 |
+
ByteLevelBPETokenizer,
|
| 97 |
+
CharBPETokenizer,
|
| 98 |
+
SentencePieceBPETokenizer,
|
| 99 |
+
SentencePieceUnigramTokenizer,
|
| 100 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/__init__.pyi
ADDED
|
@@ -0,0 +1,1800 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class AddedToken:
|
| 3 |
+
"""
|
| 4 |
+
Represents a token that can be be added to a :class:`~tokenizers.Tokenizer`.
|
| 5 |
+
It can have special options that defines the way it should behave.
|
| 6 |
+
|
| 7 |
+
Args:
|
| 8 |
+
content (:obj:`str`): The content of the token
|
| 9 |
+
|
| 10 |
+
single_word (:obj:`bool`, defaults to :obj:`False`):
|
| 11 |
+
Defines whether this token should only match single words. If :obj:`True`, this
|
| 12 |
+
token will never match inside of a word. For example the token ``ing`` would match
|
| 13 |
+
on ``tokenizing`` if this option is :obj:`False`, but not if it is :obj:`True`.
|
| 14 |
+
The notion of "`inside of a word`" is defined by the word boundaries pattern in
|
| 15 |
+
regular expressions (ie. the token should start and end with word boundaries).
|
| 16 |
+
|
| 17 |
+
lstrip (:obj:`bool`, defaults to :obj:`False`):
|
| 18 |
+
Defines whether this token should strip all potential whitespaces on its left side.
|
| 19 |
+
If :obj:`True`, this token will greedily match any whitespace on its left. For
|
| 20 |
+
example if we try to match the token ``[MASK]`` with ``lstrip=True``, in the text
|
| 21 |
+
``"I saw a [MASK]"``, we would match on ``" [MASK]"``. (Note the space on the left).
|
| 22 |
+
|
| 23 |
+
rstrip (:obj:`bool`, defaults to :obj:`False`):
|
| 24 |
+
Defines whether this token should strip all potential whitespaces on its right
|
| 25 |
+
side. If :obj:`True`, this token will greedily match any whitespace on its right.
|
| 26 |
+
It works just like :obj:`lstrip` but on the right.
|
| 27 |
+
|
| 28 |
+
normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
| 29 |
+
Defines whether this token should match against the normalized version of the input
|
| 30 |
+
text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
|
| 31 |
+
lowercasing the text, the token could be extract from the input ``"I saw a lion
|
| 32 |
+
Yesterday"``.
|
| 33 |
+
special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
| 34 |
+
Defines whether this token should be skipped when decoding.
|
| 35 |
+
|
| 36 |
+
"""
|
| 37 |
+
def __init__(self, content=None, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False):
|
| 38 |
+
pass
|
| 39 |
+
|
| 40 |
+
def __getstate__(self):
|
| 41 |
+
""" """
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
def __setstate__(self, state):
|
| 45 |
+
""" """
|
| 46 |
+
pass
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def content(self):
|
| 50 |
+
"""
|
| 51 |
+
Get the content of this :obj:`AddedToken`
|
| 52 |
+
"""
|
| 53 |
+
pass
|
| 54 |
+
|
| 55 |
+
@content.setter
|
| 56 |
+
def content(self, value):
|
| 57 |
+
"""
|
| 58 |
+
Get the content of this :obj:`AddedToken`
|
| 59 |
+
"""
|
| 60 |
+
pass
|
| 61 |
+
|
| 62 |
+
@property
|
| 63 |
+
def lstrip(self):
|
| 64 |
+
"""
|
| 65 |
+
Get the value of the :obj:`lstrip` option
|
| 66 |
+
"""
|
| 67 |
+
pass
|
| 68 |
+
|
| 69 |
+
@lstrip.setter
|
| 70 |
+
def lstrip(self, value):
|
| 71 |
+
"""
|
| 72 |
+
Get the value of the :obj:`lstrip` option
|
| 73 |
+
"""
|
| 74 |
+
pass
|
| 75 |
+
|
| 76 |
+
@property
|
| 77 |
+
def normalized(self):
|
| 78 |
+
"""
|
| 79 |
+
Get the value of the :obj:`normalized` option
|
| 80 |
+
"""
|
| 81 |
+
pass
|
| 82 |
+
|
| 83 |
+
@normalized.setter
|
| 84 |
+
def normalized(self, value):
|
| 85 |
+
"""
|
| 86 |
+
Get the value of the :obj:`normalized` option
|
| 87 |
+
"""
|
| 88 |
+
pass
|
| 89 |
+
|
| 90 |
+
@property
|
| 91 |
+
def rstrip(self):
|
| 92 |
+
"""
|
| 93 |
+
Get the value of the :obj:`rstrip` option
|
| 94 |
+
"""
|
| 95 |
+
pass
|
| 96 |
+
|
| 97 |
+
@rstrip.setter
|
| 98 |
+
def rstrip(self, value):
|
| 99 |
+
"""
|
| 100 |
+
Get the value of the :obj:`rstrip` option
|
| 101 |
+
"""
|
| 102 |
+
pass
|
| 103 |
+
|
| 104 |
+
@property
|
| 105 |
+
def single_word(self):
|
| 106 |
+
"""
|
| 107 |
+
Get the value of the :obj:`single_word` option
|
| 108 |
+
"""
|
| 109 |
+
pass
|
| 110 |
+
|
| 111 |
+
@single_word.setter
|
| 112 |
+
def single_word(self, value):
|
| 113 |
+
"""
|
| 114 |
+
Get the value of the :obj:`single_word` option
|
| 115 |
+
"""
|
| 116 |
+
pass
|
| 117 |
+
|
| 118 |
+
@property
|
| 119 |
+
def special(self):
|
| 120 |
+
"""
|
| 121 |
+
Get the value of the :obj:`special` option
|
| 122 |
+
"""
|
| 123 |
+
pass
|
| 124 |
+
|
| 125 |
+
@special.setter
|
| 126 |
+
def special(self, value):
|
| 127 |
+
"""
|
| 128 |
+
Get the value of the :obj:`special` option
|
| 129 |
+
"""
|
| 130 |
+
pass
|
| 131 |
+
|
| 132 |
+
class Encoding:
|
| 133 |
+
"""
|
| 134 |
+
The :class:`~tokenizers.Encoding` represents the output of a :class:`~tokenizers.Tokenizer`.
|
| 135 |
+
"""
|
| 136 |
+
def __init__(self):
|
| 137 |
+
pass
|
| 138 |
+
|
| 139 |
+
def __getstate__(self):
|
| 140 |
+
""" """
|
| 141 |
+
pass
|
| 142 |
+
|
| 143 |
+
def __setstate__(self, state):
|
| 144 |
+
""" """
|
| 145 |
+
pass
|
| 146 |
+
|
| 147 |
+
@property
|
| 148 |
+
def attention_mask(self):
|
| 149 |
+
"""
|
| 150 |
+
The attention mask
|
| 151 |
+
|
| 152 |
+
This indicates to the LM which tokens should be attended to, and which should not.
|
| 153 |
+
This is especially important when batching sequences, where we need to applying
|
| 154 |
+
padding.
|
| 155 |
+
|
| 156 |
+
Returns:
|
| 157 |
+
:obj:`List[int]`: The attention mask
|
| 158 |
+
"""
|
| 159 |
+
pass
|
| 160 |
+
|
| 161 |
+
@attention_mask.setter
|
| 162 |
+
def attention_mask(self, value):
|
| 163 |
+
"""
|
| 164 |
+
The attention mask
|
| 165 |
+
|
| 166 |
+
This indicates to the LM which tokens should be attended to, and which should not.
|
| 167 |
+
This is especially important when batching sequences, where we need to applying
|
| 168 |
+
padding.
|
| 169 |
+
|
| 170 |
+
Returns:
|
| 171 |
+
:obj:`List[int]`: The attention mask
|
| 172 |
+
"""
|
| 173 |
+
pass
|
| 174 |
+
|
| 175 |
+
def char_to_token(self, char_pos, sequence_index=0):
|
| 176 |
+
"""
|
| 177 |
+
Get the token that contains the char at the given position in the input sequence.
|
| 178 |
+
|
| 179 |
+
Args:
|
| 180 |
+
char_pos (:obj:`int`):
|
| 181 |
+
The position of a char in the input string
|
| 182 |
+
sequence_index (:obj:`int`, defaults to :obj:`0`):
|
| 183 |
+
The index of the sequence that contains the target char
|
| 184 |
+
|
| 185 |
+
Returns:
|
| 186 |
+
:obj:`int`: The index of the token that contains this char in the encoded sequence
|
| 187 |
+
"""
|
| 188 |
+
pass
|
| 189 |
+
|
| 190 |
+
def char_to_word(self, char_pos, sequence_index=0):
|
| 191 |
+
"""
|
| 192 |
+
Get the word that contains the char at the given position in the input sequence.
|
| 193 |
+
|
| 194 |
+
Args:
|
| 195 |
+
char_pos (:obj:`int`):
|
| 196 |
+
The position of a char in the input string
|
| 197 |
+
sequence_index (:obj:`int`, defaults to :obj:`0`):
|
| 198 |
+
The index of the sequence that contains the target char
|
| 199 |
+
|
| 200 |
+
Returns:
|
| 201 |
+
:obj:`int`: The index of the word that contains this char in the input sequence
|
| 202 |
+
"""
|
| 203 |
+
pass
|
| 204 |
+
|
| 205 |
+
@property
|
| 206 |
+
def ids(self):
|
| 207 |
+
"""
|
| 208 |
+
The generated IDs
|
| 209 |
+
|
| 210 |
+
The IDs are the main input to a Language Model. They are the token indices,
|
| 211 |
+
the numerical representations that a LM understands.
|
| 212 |
+
|
| 213 |
+
Returns:
|
| 214 |
+
:obj:`List[int]`: The list of IDs
|
| 215 |
+
"""
|
| 216 |
+
pass
|
| 217 |
+
|
| 218 |
+
@ids.setter
|
| 219 |
+
def ids(self, value):
|
| 220 |
+
"""
|
| 221 |
+
The generated IDs
|
| 222 |
+
|
| 223 |
+
The IDs are the main input to a Language Model. They are the token indices,
|
| 224 |
+
the numerical representations that a LM understands.
|
| 225 |
+
|
| 226 |
+
Returns:
|
| 227 |
+
:obj:`List[int]`: The list of IDs
|
| 228 |
+
"""
|
| 229 |
+
pass
|
| 230 |
+
|
| 231 |
+
@staticmethod
|
| 232 |
+
def merge(encodings, growing_offsets=True):
|
| 233 |
+
"""
|
| 234 |
+
Merge the list of encodings into one final :class:`~tokenizers.Encoding`
|
| 235 |
+
|
| 236 |
+
Args:
|
| 237 |
+
encodings (A :obj:`List` of :class:`~tokenizers.Encoding`):
|
| 238 |
+
The list of encodings that should be merged in one
|
| 239 |
+
|
| 240 |
+
growing_offsets (:obj:`bool`, defaults to :obj:`True`):
|
| 241 |
+
Whether the offsets should accumulate while merging
|
| 242 |
+
|
| 243 |
+
Returns:
|
| 244 |
+
:class:`~tokenizers.Encoding`: The resulting Encoding
|
| 245 |
+
"""
|
| 246 |
+
pass
|
| 247 |
+
|
| 248 |
+
@property
|
| 249 |
+
def n_sequences(self):
|
| 250 |
+
"""
|
| 251 |
+
The number of sequences represented
|
| 252 |
+
|
| 253 |
+
Returns:
|
| 254 |
+
:obj:`int`: The number of sequences in this :class:`~tokenizers.Encoding`
|
| 255 |
+
"""
|
| 256 |
+
pass
|
| 257 |
+
|
| 258 |
+
@n_sequences.setter
|
| 259 |
+
def n_sequences(self, value):
|
| 260 |
+
"""
|
| 261 |
+
The number of sequences represented
|
| 262 |
+
|
| 263 |
+
Returns:
|
| 264 |
+
:obj:`int`: The number of sequences in this :class:`~tokenizers.Encoding`
|
| 265 |
+
"""
|
| 266 |
+
pass
|
| 267 |
+
|
| 268 |
+
@property
|
| 269 |
+
def offsets(self):
|
| 270 |
+
"""
|
| 271 |
+
The offsets associated to each token
|
| 272 |
+
|
| 273 |
+
These offsets let's you slice the input string, and thus retrieve the original
|
| 274 |
+
part that led to producing the corresponding token.
|
| 275 |
+
|
| 276 |
+
Returns:
|
| 277 |
+
A :obj:`List` of :obj:`Tuple[int, int]`: The list of offsets
|
| 278 |
+
"""
|
| 279 |
+
pass
|
| 280 |
+
|
| 281 |
+
@offsets.setter
|
| 282 |
+
def offsets(self, value):
|
| 283 |
+
"""
|
| 284 |
+
The offsets associated to each token
|
| 285 |
+
|
| 286 |
+
These offsets let's you slice the input string, and thus retrieve the original
|
| 287 |
+
part that led to producing the corresponding token.
|
| 288 |
+
|
| 289 |
+
Returns:
|
| 290 |
+
A :obj:`List` of :obj:`Tuple[int, int]`: The list of offsets
|
| 291 |
+
"""
|
| 292 |
+
pass
|
| 293 |
+
|
| 294 |
+
@property
|
| 295 |
+
def overflowing(self):
|
| 296 |
+
"""
|
| 297 |
+
A :obj:`List` of overflowing :class:`~tokenizers.Encoding`
|
| 298 |
+
|
| 299 |
+
When using truncation, the :class:`~tokenizers.Tokenizer` takes care of splitting
|
| 300 |
+
the output into as many pieces as required to match the specified maximum length.
|
| 301 |
+
This field lets you retrieve all the subsequent pieces.
|
| 302 |
+
|
| 303 |
+
When you use pairs of sequences, the overflowing pieces will contain enough
|
| 304 |
+
variations to cover all the possible combinations, while respecting the provided
|
| 305 |
+
maximum length.
|
| 306 |
+
"""
|
| 307 |
+
pass
|
| 308 |
+
|
| 309 |
+
@overflowing.setter
|
| 310 |
+
def overflowing(self, value):
|
| 311 |
+
"""
|
| 312 |
+
A :obj:`List` of overflowing :class:`~tokenizers.Encoding`
|
| 313 |
+
|
| 314 |
+
When using truncation, the :class:`~tokenizers.Tokenizer` takes care of splitting
|
| 315 |
+
the output into as many pieces as required to match the specified maximum length.
|
| 316 |
+
This field lets you retrieve all the subsequent pieces.
|
| 317 |
+
|
| 318 |
+
When you use pairs of sequences, the overflowing pieces will contain enough
|
| 319 |
+
variations to cover all the possible combinations, while respecting the provided
|
| 320 |
+
maximum length.
|
| 321 |
+
"""
|
| 322 |
+
pass
|
| 323 |
+
|
| 324 |
+
def pad(self, length, direction="right", pad_id=0, pad_type_id=0, pad_token="[PAD]"):
|
| 325 |
+
"""
|
| 326 |
+
Pad the :class:`~tokenizers.Encoding` at the given length
|
| 327 |
+
|
| 328 |
+
Args:
|
| 329 |
+
length (:obj:`int`):
|
| 330 |
+
The desired length
|
| 331 |
+
|
| 332 |
+
direction: (:obj:`str`, defaults to :obj:`right`):
|
| 333 |
+
The expected padding direction. Can be either :obj:`right` or :obj:`left`
|
| 334 |
+
|
| 335 |
+
pad_id (:obj:`int`, defaults to :obj:`0`):
|
| 336 |
+
The ID corresponding to the padding token
|
| 337 |
+
|
| 338 |
+
pad_type_id (:obj:`int`, defaults to :obj:`0`):
|
| 339 |
+
The type ID corresponding to the padding token
|
| 340 |
+
|
| 341 |
+
pad_token (:obj:`str`, defaults to `[PAD]`):
|
| 342 |
+
The pad token to use
|
| 343 |
+
"""
|
| 344 |
+
pass
|
| 345 |
+
|
| 346 |
+
@property
|
| 347 |
+
def sequence_ids(self):
|
| 348 |
+
"""
|
| 349 |
+
The generated sequence indices.
|
| 350 |
+
|
| 351 |
+
They represent the index of the input sequence associated to each token.
|
| 352 |
+
The sequence id can be None if the token is not related to any input sequence,
|
| 353 |
+
like for example with special tokens.
|
| 354 |
+
|
| 355 |
+
Returns:
|
| 356 |
+
A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index.
|
| 357 |
+
"""
|
| 358 |
+
pass
|
| 359 |
+
|
| 360 |
+
@sequence_ids.setter
|
| 361 |
+
def sequence_ids(self, value):
|
| 362 |
+
"""
|
| 363 |
+
The generated sequence indices.
|
| 364 |
+
|
| 365 |
+
They represent the index of the input sequence associated to each token.
|
| 366 |
+
The sequence id can be None if the token is not related to any input sequence,
|
| 367 |
+
like for example with special tokens.
|
| 368 |
+
|
| 369 |
+
Returns:
|
| 370 |
+
A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index.
|
| 371 |
+
"""
|
| 372 |
+
pass
|
| 373 |
+
|
| 374 |
+
def set_sequence_id(self, sequence_id):
|
| 375 |
+
"""
|
| 376 |
+
Set the given sequence index
|
| 377 |
+
|
| 378 |
+
Set the given sequence index for the whole range of tokens contained in this
|
| 379 |
+
:class:`~tokenizers.Encoding`.
|
| 380 |
+
"""
|
| 381 |
+
pass
|
| 382 |
+
|
| 383 |
+
@property
|
| 384 |
+
def special_tokens_mask(self):
|
| 385 |
+
"""
|
| 386 |
+
The special token mask
|
| 387 |
+
|
| 388 |
+
This indicates which tokens are special tokens, and which are not.
|
| 389 |
+
|
| 390 |
+
Returns:
|
| 391 |
+
:obj:`List[int]`: The special tokens mask
|
| 392 |
+
"""
|
| 393 |
+
pass
|
| 394 |
+
|
| 395 |
+
@special_tokens_mask.setter
|
| 396 |
+
def special_tokens_mask(self, value):
|
| 397 |
+
"""
|
| 398 |
+
The special token mask
|
| 399 |
+
|
| 400 |
+
This indicates which tokens are special tokens, and which are not.
|
| 401 |
+
|
| 402 |
+
Returns:
|
| 403 |
+
:obj:`List[int]`: The special tokens mask
|
| 404 |
+
"""
|
| 405 |
+
pass
|
| 406 |
+
|
| 407 |
+
def token_to_chars(self, token_index):
|
| 408 |
+
"""
|
| 409 |
+
Get the offsets of the token at the given index.
|
| 410 |
+
|
| 411 |
+
The returned offsets are related to the input sequence that contains the
|
| 412 |
+
token. In order to determine in which input sequence it belongs, you
|
| 413 |
+
must call :meth:`~tokenizers.Encoding.token_to_sequence()`.
|
| 414 |
+
|
| 415 |
+
Args:
|
| 416 |
+
token_index (:obj:`int`):
|
| 417 |
+
The index of a token in the encoded sequence.
|
| 418 |
+
|
| 419 |
+
Returns:
|
| 420 |
+
:obj:`Tuple[int, int]`: The token offsets :obj:`(first, last + 1)`
|
| 421 |
+
"""
|
| 422 |
+
pass
|
| 423 |
+
|
| 424 |
+
def token_to_sequence(self, token_index):
|
| 425 |
+
"""
|
| 426 |
+
Get the index of the sequence represented by the given token.
|
| 427 |
+
|
| 428 |
+
In the general use case, this method returns :obj:`0` for a single sequence or
|
| 429 |
+
the first sequence of a pair, and :obj:`1` for the second sequence of a pair
|
| 430 |
+
|
| 431 |
+
Args:
|
| 432 |
+
token_index (:obj:`int`):
|
| 433 |
+
The index of a token in the encoded sequence.
|
| 434 |
+
|
| 435 |
+
Returns:
|
| 436 |
+
:obj:`int`: The sequence id of the given token
|
| 437 |
+
"""
|
| 438 |
+
pass
|
| 439 |
+
|
| 440 |
+
def token_to_word(self, token_index):
|
| 441 |
+
"""
|
| 442 |
+
Get the index of the word that contains the token in one of the input sequences.
|
| 443 |
+
|
| 444 |
+
The returned word index is related to the input sequence that contains
|
| 445 |
+
the token. In order to determine in which input sequence it belongs, you
|
| 446 |
+
must call :meth:`~tokenizers.Encoding.token_to_sequence()`.
|
| 447 |
+
|
| 448 |
+
Args:
|
| 449 |
+
token_index (:obj:`int`):
|
| 450 |
+
The index of a token in the encoded sequence.
|
| 451 |
+
|
| 452 |
+
Returns:
|
| 453 |
+
:obj:`int`: The index of the word in the relevant input sequence.
|
| 454 |
+
"""
|
| 455 |
+
pass
|
| 456 |
+
|
| 457 |
+
@property
|
| 458 |
+
def tokens(self):
|
| 459 |
+
"""
|
| 460 |
+
The generated tokens
|
| 461 |
+
|
| 462 |
+
They are the string representation of the IDs.
|
| 463 |
+
|
| 464 |
+
Returns:
|
| 465 |
+
:obj:`List[str]`: The list of tokens
|
| 466 |
+
"""
|
| 467 |
+
pass
|
| 468 |
+
|
| 469 |
+
@tokens.setter
|
| 470 |
+
def tokens(self, value):
|
| 471 |
+
"""
|
| 472 |
+
The generated tokens
|
| 473 |
+
|
| 474 |
+
They are the string representation of the IDs.
|
| 475 |
+
|
| 476 |
+
Returns:
|
| 477 |
+
:obj:`List[str]`: The list of tokens
|
| 478 |
+
"""
|
| 479 |
+
pass
|
| 480 |
+
|
| 481 |
+
def truncate(self, max_length, stride=0, direction="right"):
|
| 482 |
+
"""
|
| 483 |
+
Truncate the :class:`~tokenizers.Encoding` at the given length
|
| 484 |
+
|
| 485 |
+
If this :class:`~tokenizers.Encoding` represents multiple sequences, when truncating
|
| 486 |
+
this information is lost. It will be considered as representing a single sequence.
|
| 487 |
+
|
| 488 |
+
Args:
|
| 489 |
+
max_length (:obj:`int`):
|
| 490 |
+
The desired length
|
| 491 |
+
|
| 492 |
+
stride (:obj:`int`, defaults to :obj:`0`):
|
| 493 |
+
The length of previous content to be included in each overflowing piece
|
| 494 |
+
|
| 495 |
+
direction (:obj:`str`, defaults to :obj:`right`):
|
| 496 |
+
Truncate direction
|
| 497 |
+
"""
|
| 498 |
+
pass
|
| 499 |
+
|
| 500 |
+
@property
|
| 501 |
+
def type_ids(self):
|
| 502 |
+
"""
|
| 503 |
+
The generated type IDs
|
| 504 |
+
|
| 505 |
+
Generally used for tasks like sequence classification or question answering,
|
| 506 |
+
these tokens let the LM know which input sequence corresponds to each tokens.
|
| 507 |
+
|
| 508 |
+
Returns:
|
| 509 |
+
:obj:`List[int]`: The list of type ids
|
| 510 |
+
"""
|
| 511 |
+
pass
|
| 512 |
+
|
| 513 |
+
@type_ids.setter
|
| 514 |
+
def type_ids(self, value):
|
| 515 |
+
"""
|
| 516 |
+
The generated type IDs
|
| 517 |
+
|
| 518 |
+
Generally used for tasks like sequence classification or question answering,
|
| 519 |
+
these tokens let the LM know which input sequence corresponds to each tokens.
|
| 520 |
+
|
| 521 |
+
Returns:
|
| 522 |
+
:obj:`List[int]`: The list of type ids
|
| 523 |
+
"""
|
| 524 |
+
pass
|
| 525 |
+
|
| 526 |
+
@property
|
| 527 |
+
def word_ids(self):
|
| 528 |
+
"""
|
| 529 |
+
The generated word indices.
|
| 530 |
+
|
| 531 |
+
They represent the index of the word associated to each token.
|
| 532 |
+
When the input is pre-tokenized, they correspond to the ID of the given input label,
|
| 533 |
+
otherwise they correspond to the words indices as defined by the
|
| 534 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
|
| 535 |
+
|
| 536 |
+
For special tokens and such (any token that was generated from something that was
|
| 537 |
+
not part of the input), the output is :obj:`None`
|
| 538 |
+
|
| 539 |
+
Returns:
|
| 540 |
+
A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
|
| 541 |
+
"""
|
| 542 |
+
pass
|
| 543 |
+
|
| 544 |
+
@word_ids.setter
|
| 545 |
+
def word_ids(self, value):
|
| 546 |
+
"""
|
| 547 |
+
The generated word indices.
|
| 548 |
+
|
| 549 |
+
They represent the index of the word associated to each token.
|
| 550 |
+
When the input is pre-tokenized, they correspond to the ID of the given input label,
|
| 551 |
+
otherwise they correspond to the words indices as defined by the
|
| 552 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
|
| 553 |
+
|
| 554 |
+
For special tokens and such (any token that was generated from something that was
|
| 555 |
+
not part of the input), the output is :obj:`None`
|
| 556 |
+
|
| 557 |
+
Returns:
|
| 558 |
+
A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
|
| 559 |
+
"""
|
| 560 |
+
pass
|
| 561 |
+
|
| 562 |
+
def word_to_chars(self, word_index, sequence_index=0):
|
| 563 |
+
"""
|
| 564 |
+
Get the offsets of the word at the given index in one of the input sequences.
|
| 565 |
+
|
| 566 |
+
Args:
|
| 567 |
+
word_index (:obj:`int`):
|
| 568 |
+
The index of a word in one of the input sequences.
|
| 569 |
+
sequence_index (:obj:`int`, defaults to :obj:`0`):
|
| 570 |
+
The index of the sequence that contains the target word
|
| 571 |
+
|
| 572 |
+
Returns:
|
| 573 |
+
:obj:`Tuple[int, int]`: The range of characters (span) :obj:`(first, last + 1)`
|
| 574 |
+
"""
|
| 575 |
+
pass
|
| 576 |
+
|
| 577 |
+
def word_to_tokens(self, word_index, sequence_index=0):
|
| 578 |
+
"""
|
| 579 |
+
Get the encoded tokens corresponding to the word at the given index
|
| 580 |
+
in one of the input sequences.
|
| 581 |
+
|
| 582 |
+
Args:
|
| 583 |
+
word_index (:obj:`int`):
|
| 584 |
+
The index of a word in one of the input sequences.
|
| 585 |
+
sequence_index (:obj:`int`, defaults to :obj:`0`):
|
| 586 |
+
The index of the sequence that contains the target word
|
| 587 |
+
|
| 588 |
+
Returns:
|
| 589 |
+
:obj:`Tuple[int, int]`: The range of tokens: :obj:`(first, last + 1)`
|
| 590 |
+
"""
|
| 591 |
+
pass
|
| 592 |
+
|
| 593 |
+
@property
|
| 594 |
+
def words(self):
|
| 595 |
+
"""
|
| 596 |
+
The generated word indices.
|
| 597 |
+
|
| 598 |
+
.. warning::
|
| 599 |
+
This is deprecated and will be removed in a future version.
|
| 600 |
+
Please use :obj:`~tokenizers.Encoding.word_ids` instead.
|
| 601 |
+
|
| 602 |
+
They represent the index of the word associated to each token.
|
| 603 |
+
When the input is pre-tokenized, they correspond to the ID of the given input label,
|
| 604 |
+
otherwise they correspond to the words indices as defined by the
|
| 605 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
|
| 606 |
+
|
| 607 |
+
For special tokens and such (any token that was generated from something that was
|
| 608 |
+
not part of the input), the output is :obj:`None`
|
| 609 |
+
|
| 610 |
+
Returns:
|
| 611 |
+
A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
|
| 612 |
+
"""
|
| 613 |
+
pass
|
| 614 |
+
|
| 615 |
+
@words.setter
|
| 616 |
+
def words(self, value):
|
| 617 |
+
"""
|
| 618 |
+
The generated word indices.
|
| 619 |
+
|
| 620 |
+
.. warning::
|
| 621 |
+
This is deprecated and will be removed in a future version.
|
| 622 |
+
Please use :obj:`~tokenizers.Encoding.word_ids` instead.
|
| 623 |
+
|
| 624 |
+
They represent the index of the word associated to each token.
|
| 625 |
+
When the input is pre-tokenized, they correspond to the ID of the given input label,
|
| 626 |
+
otherwise they correspond to the words indices as defined by the
|
| 627 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
|
| 628 |
+
|
| 629 |
+
For special tokens and such (any token that was generated from something that was
|
| 630 |
+
not part of the input), the output is :obj:`None`
|
| 631 |
+
|
| 632 |
+
Returns:
|
| 633 |
+
A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
|
| 634 |
+
"""
|
| 635 |
+
pass
|
| 636 |
+
|
| 637 |
+
class NormalizedString:
|
| 638 |
+
"""
|
| 639 |
+
NormalizedString
|
| 640 |
+
|
| 641 |
+
A NormalizedString takes care of modifying an "original" string, to obtain a "normalized" one.
|
| 642 |
+
While making all the requested modifications, it keeps track of the alignment information
|
| 643 |
+
between the two versions of the string.
|
| 644 |
+
|
| 645 |
+
Args:
|
| 646 |
+
sequence: str:
|
| 647 |
+
The string sequence used to initialize this NormalizedString
|
| 648 |
+
"""
|
| 649 |
+
def __init__(self, sequence):
|
| 650 |
+
pass
|
| 651 |
+
|
| 652 |
+
def __getitem__(self, key):
|
| 653 |
+
"""
|
| 654 |
+
Return self[key].
|
| 655 |
+
"""
|
| 656 |
+
pass
|
| 657 |
+
|
| 658 |
+
def __getstate__(self, /):
|
| 659 |
+
"""
|
| 660 |
+
Helper for pickle.
|
| 661 |
+
"""
|
| 662 |
+
pass
|
| 663 |
+
|
| 664 |
+
def append(self, s):
|
| 665 |
+
"""
|
| 666 |
+
Append the given sequence to the string
|
| 667 |
+
"""
|
| 668 |
+
pass
|
| 669 |
+
|
| 670 |
+
def clear(self):
|
| 671 |
+
"""
|
| 672 |
+
Clears the string
|
| 673 |
+
"""
|
| 674 |
+
pass
|
| 675 |
+
|
| 676 |
+
def filter(self, func):
|
| 677 |
+
"""
|
| 678 |
+
Filter each character of the string using the given func
|
| 679 |
+
"""
|
| 680 |
+
pass
|
| 681 |
+
|
| 682 |
+
def for_each(self, func):
|
| 683 |
+
"""
|
| 684 |
+
Calls the given function for each character of the string
|
| 685 |
+
"""
|
| 686 |
+
pass
|
| 687 |
+
|
| 688 |
+
def lowercase(self):
|
| 689 |
+
"""
|
| 690 |
+
Lowercase the string
|
| 691 |
+
"""
|
| 692 |
+
pass
|
| 693 |
+
|
| 694 |
+
def lstrip(self):
|
| 695 |
+
"""
|
| 696 |
+
Strip the left of the string
|
| 697 |
+
"""
|
| 698 |
+
pass
|
| 699 |
+
|
| 700 |
+
def map(self, func):
|
| 701 |
+
"""
|
| 702 |
+
Calls the given function for each character of the string
|
| 703 |
+
|
| 704 |
+
Replaces each character of the string using the returned value. Each
|
| 705 |
+
returned value **must** be a str of length 1 (ie a character).
|
| 706 |
+
"""
|
| 707 |
+
pass
|
| 708 |
+
|
| 709 |
+
def nfc(self):
|
| 710 |
+
"""
|
| 711 |
+
Runs the NFC normalization
|
| 712 |
+
"""
|
| 713 |
+
pass
|
| 714 |
+
|
| 715 |
+
def nfd(self):
|
| 716 |
+
"""
|
| 717 |
+
Runs the NFD normalization
|
| 718 |
+
"""
|
| 719 |
+
pass
|
| 720 |
+
|
| 721 |
+
def nfkc(self):
|
| 722 |
+
"""
|
| 723 |
+
Runs the NFKC normalization
|
| 724 |
+
"""
|
| 725 |
+
pass
|
| 726 |
+
|
| 727 |
+
def nfkd(self):
|
| 728 |
+
"""
|
| 729 |
+
Runs the NFKD normalization
|
| 730 |
+
"""
|
| 731 |
+
pass
|
| 732 |
+
|
| 733 |
+
@property
|
| 734 |
+
def normalized(self):
|
| 735 |
+
"""
|
| 736 |
+
The normalized part of the string
|
| 737 |
+
"""
|
| 738 |
+
pass
|
| 739 |
+
|
| 740 |
+
@normalized.setter
|
| 741 |
+
def normalized(self, value):
|
| 742 |
+
"""
|
| 743 |
+
The normalized part of the string
|
| 744 |
+
"""
|
| 745 |
+
pass
|
| 746 |
+
|
| 747 |
+
@property
|
| 748 |
+
def original(self):
|
| 749 |
+
""" """
|
| 750 |
+
pass
|
| 751 |
+
|
| 752 |
+
@original.setter
|
| 753 |
+
def original(self, value):
|
| 754 |
+
""" """
|
| 755 |
+
pass
|
| 756 |
+
|
| 757 |
+
def prepend(self, s):
|
| 758 |
+
"""
|
| 759 |
+
Prepend the given sequence to the string
|
| 760 |
+
"""
|
| 761 |
+
pass
|
| 762 |
+
|
| 763 |
+
def replace(self, pattern, content):
|
| 764 |
+
"""
|
| 765 |
+
Replace the content of the given pattern with the provided content
|
| 766 |
+
|
| 767 |
+
Args:
|
| 768 |
+
pattern: Pattern:
|
| 769 |
+
A pattern used to match the string. Usually a string or a Regex
|
| 770 |
+
|
| 771 |
+
content: str:
|
| 772 |
+
The content to be used as replacement
|
| 773 |
+
"""
|
| 774 |
+
pass
|
| 775 |
+
|
| 776 |
+
def rstrip(self):
|
| 777 |
+
"""
|
| 778 |
+
Strip the right of the string
|
| 779 |
+
"""
|
| 780 |
+
pass
|
| 781 |
+
|
| 782 |
+
def slice(self, range):
|
| 783 |
+
"""
|
| 784 |
+
Slice the string using the given range
|
| 785 |
+
"""
|
| 786 |
+
pass
|
| 787 |
+
|
| 788 |
+
def split(self, pattern, behavior):
|
| 789 |
+
"""
|
| 790 |
+
Split the NormalizedString using the given pattern and the specified behavior
|
| 791 |
+
|
| 792 |
+
Args:
|
| 793 |
+
pattern: Pattern:
|
| 794 |
+
A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`
|
| 795 |
+
|
| 796 |
+
behavior: SplitDelimiterBehavior:
|
| 797 |
+
The behavior to use when splitting.
|
| 798 |
+
Choices: "removed", "isolated", "merged_with_previous", "merged_with_next",
|
| 799 |
+
"contiguous"
|
| 800 |
+
|
| 801 |
+
Returns:
|
| 802 |
+
A list of NormalizedString, representing each split
|
| 803 |
+
"""
|
| 804 |
+
pass
|
| 805 |
+
|
| 806 |
+
def strip(self):
|
| 807 |
+
"""
|
| 808 |
+
Strip both ends of the string
|
| 809 |
+
"""
|
| 810 |
+
pass
|
| 811 |
+
|
| 812 |
+
def uppercase(self):
|
| 813 |
+
"""
|
| 814 |
+
Uppercase the string
|
| 815 |
+
"""
|
| 816 |
+
pass
|
| 817 |
+
|
| 818 |
+
class PreTokenizedString:
|
| 819 |
+
"""
|
| 820 |
+
PreTokenizedString
|
| 821 |
+
|
| 822 |
+
Wrapper over a string, that provides a way to normalize, pre-tokenize, tokenize the
|
| 823 |
+
underlying string, while keeping track of the alignment information (offsets).
|
| 824 |
+
|
| 825 |
+
The PreTokenizedString manages what we call `splits`. Each split represents a substring
|
| 826 |
+
which is a subpart of the original string, with the relevant offsets and tokens.
|
| 827 |
+
|
| 828 |
+
When calling one of the methods used to modify the PreTokenizedString (namely one of
|
| 829 |
+
`split`, `normalize` or `tokenize), only the `splits` that don't have any associated
|
| 830 |
+
tokens will get modified.
|
| 831 |
+
|
| 832 |
+
Args:
|
| 833 |
+
sequence: str:
|
| 834 |
+
The string sequence used to initialize this PreTokenizedString
|
| 835 |
+
"""
|
| 836 |
+
def __init__(self, sequence):
|
| 837 |
+
pass
|
| 838 |
+
|
| 839 |
+
def __getstate__(self, /):
|
| 840 |
+
"""
|
| 841 |
+
Helper for pickle.
|
| 842 |
+
"""
|
| 843 |
+
pass
|
| 844 |
+
|
| 845 |
+
def get_splits(self, offset_referential="original", offset_type="char"):
|
| 846 |
+
"""
|
| 847 |
+
Get the splits currently managed by the PreTokenizedString
|
| 848 |
+
|
| 849 |
+
Args:
|
| 850 |
+
offset_referential: :obj:`str`
|
| 851 |
+
Whether the returned splits should have offsets expressed relative
|
| 852 |
+
to the original string, or the normalized one. choices: "original", "normalized".
|
| 853 |
+
|
| 854 |
+
offset_type: :obj:`str`
|
| 855 |
+
Whether the returned splits should have offsets expressed in bytes or chars.
|
| 856 |
+
When slicing an str, we usually want to use chars, which is the default value.
|
| 857 |
+
Now in some cases it might be interesting to get these offsets expressed in bytes,
|
| 858 |
+
so it is possible to change this here.
|
| 859 |
+
choices: "char", "bytes"
|
| 860 |
+
|
| 861 |
+
Returns
|
| 862 |
+
A list of splits
|
| 863 |
+
"""
|
| 864 |
+
pass
|
| 865 |
+
|
| 866 |
+
def normalize(self, func):
|
| 867 |
+
"""
|
| 868 |
+
Normalize each split of the `PreTokenizedString` using the given `func`
|
| 869 |
+
|
| 870 |
+
Args:
|
| 871 |
+
func: Callable[[NormalizedString], None]:
|
| 872 |
+
The function used to normalize each underlying split. This function
|
| 873 |
+
does not need to return anything, just calling the methods on the provided
|
| 874 |
+
NormalizedString allow its modification.
|
| 875 |
+
"""
|
| 876 |
+
pass
|
| 877 |
+
|
| 878 |
+
def split(self, func):
|
| 879 |
+
"""
|
| 880 |
+
Split the PreTokenizedString using the given `func`
|
| 881 |
+
|
| 882 |
+
Args:
|
| 883 |
+
func: Callable[[index, NormalizedString], List[NormalizedString]]:
|
| 884 |
+
The function used to split each underlying split.
|
| 885 |
+
It is expected to return a list of `NormalizedString`, that represent the new
|
| 886 |
+
splits. If the given `NormalizedString` does not need any splitting, we can
|
| 887 |
+
just return it directly.
|
| 888 |
+
In order for the offsets to be tracked accurately, any returned `NormalizedString`
|
| 889 |
+
should come from calling either `.split` or `.slice` on the received one.
|
| 890 |
+
"""
|
| 891 |
+
pass
|
| 892 |
+
|
| 893 |
+
def to_encoding(self, type_id=0, word_idx=None):
|
| 894 |
+
"""
|
| 895 |
+
Return an Encoding generated from this PreTokenizedString
|
| 896 |
+
|
| 897 |
+
Args:
|
| 898 |
+
type_id: int = 0:
|
| 899 |
+
The type_id to be used on the generated Encoding.
|
| 900 |
+
|
| 901 |
+
word_idx: Optional[int] = None:
|
| 902 |
+
An optional word index to be used for each token of this Encoding. If provided,
|
| 903 |
+
all the word indices in the generated Encoding will use this value, instead
|
| 904 |
+
of the one automatically tracked during pre-tokenization.
|
| 905 |
+
|
| 906 |
+
Returns:
|
| 907 |
+
An Encoding
|
| 908 |
+
"""
|
| 909 |
+
pass
|
| 910 |
+
|
| 911 |
+
def tokenize(self, func):
|
| 912 |
+
"""
|
| 913 |
+
Tokenize each split of the `PreTokenizedString` using the given `func`
|
| 914 |
+
|
| 915 |
+
Args:
|
| 916 |
+
func: Callable[[str], List[Token]]:
|
| 917 |
+
The function used to tokenize each underlying split. This function must return
|
| 918 |
+
a list of Token generated from the input str.
|
| 919 |
+
"""
|
| 920 |
+
pass
|
| 921 |
+
|
| 922 |
+
class Regex:
|
| 923 |
+
"""
|
| 924 |
+
Instantiate a new Regex with the given pattern
|
| 925 |
+
"""
|
| 926 |
+
def __init__(self, pattern):
|
| 927 |
+
pass
|
| 928 |
+
|
| 929 |
+
def __getstate__(self, /):
|
| 930 |
+
"""
|
| 931 |
+
Helper for pickle.
|
| 932 |
+
"""
|
| 933 |
+
pass
|
| 934 |
+
|
| 935 |
+
class Token:
|
| 936 |
+
def __init__(self, id, value, offsets):
|
| 937 |
+
pass
|
| 938 |
+
|
| 939 |
+
def __getstate__(self, /):
|
| 940 |
+
"""
|
| 941 |
+
Helper for pickle.
|
| 942 |
+
"""
|
| 943 |
+
pass
|
| 944 |
+
|
| 945 |
+
def as_tuple(self):
|
| 946 |
+
""" """
|
| 947 |
+
pass
|
| 948 |
+
|
| 949 |
+
@property
|
| 950 |
+
def id(self):
|
| 951 |
+
""" """
|
| 952 |
+
pass
|
| 953 |
+
|
| 954 |
+
@id.setter
|
| 955 |
+
def id(self, value):
|
| 956 |
+
""" """
|
| 957 |
+
pass
|
| 958 |
+
|
| 959 |
+
@property
|
| 960 |
+
def offsets(self):
|
| 961 |
+
""" """
|
| 962 |
+
pass
|
| 963 |
+
|
| 964 |
+
@offsets.setter
|
| 965 |
+
def offsets(self, value):
|
| 966 |
+
""" """
|
| 967 |
+
pass
|
| 968 |
+
|
| 969 |
+
@property
|
| 970 |
+
def value(self):
|
| 971 |
+
""" """
|
| 972 |
+
pass
|
| 973 |
+
|
| 974 |
+
@value.setter
|
| 975 |
+
def value(self, value):
|
| 976 |
+
""" """
|
| 977 |
+
pass
|
| 978 |
+
|
| 979 |
+
class Tokenizer:
|
| 980 |
+
"""
|
| 981 |
+
A :obj:`Tokenizer` works as a pipeline. It processes some raw text as input
|
| 982 |
+
and outputs an :class:`~tokenizers.Encoding`.
|
| 983 |
+
|
| 984 |
+
Args:
|
| 985 |
+
model (:class:`~tokenizers.models.Model`):
|
| 986 |
+
The core algorithm that this :obj:`Tokenizer` should be using.
|
| 987 |
+
|
| 988 |
+
"""
|
| 989 |
+
def __init__(self, model):
|
| 990 |
+
pass
|
| 991 |
+
|
| 992 |
+
def __getnewargs__(self):
|
| 993 |
+
""" """
|
| 994 |
+
pass
|
| 995 |
+
|
| 996 |
+
def __getstate__(self):
|
| 997 |
+
""" """
|
| 998 |
+
pass
|
| 999 |
+
|
| 1000 |
+
def __setstate__(self, state):
|
| 1001 |
+
""" """
|
| 1002 |
+
pass
|
| 1003 |
+
|
| 1004 |
+
def add_special_tokens(self, tokens):
|
| 1005 |
+
"""
|
| 1006 |
+
Add the given special tokens to the Tokenizer.
|
| 1007 |
+
|
| 1008 |
+
If these tokens are already part of the vocabulary, it just let the Tokenizer know about
|
| 1009 |
+
them. If they don't exist, the Tokenizer creates them, giving them a new id.
|
| 1010 |
+
|
| 1011 |
+
These special tokens will never be processed by the model (ie won't be split into
|
| 1012 |
+
multiple tokens), and they can be removed from the output when decoding.
|
| 1013 |
+
|
| 1014 |
+
Args:
|
| 1015 |
+
tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`):
|
| 1016 |
+
The list of special tokens we want to add to the vocabulary. Each token can either
|
| 1017 |
+
be a string or an instance of :class:`~tokenizers.AddedToken` for more
|
| 1018 |
+
customization.
|
| 1019 |
+
|
| 1020 |
+
Returns:
|
| 1021 |
+
:obj:`int`: The number of tokens that were created in the vocabulary
|
| 1022 |
+
"""
|
| 1023 |
+
pass
|
| 1024 |
+
|
| 1025 |
+
def add_tokens(self, tokens):
|
| 1026 |
+
"""
|
| 1027 |
+
Add the given tokens to the vocabulary
|
| 1028 |
+
|
| 1029 |
+
The given tokens are added only if they don't already exist in the vocabulary.
|
| 1030 |
+
Each token then gets a new attributed id.
|
| 1031 |
+
|
| 1032 |
+
Args:
|
| 1033 |
+
tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`):
|
| 1034 |
+
The list of tokens we want to add to the vocabulary. Each token can be either a
|
| 1035 |
+
string or an instance of :class:`~tokenizers.AddedToken` for more customization.
|
| 1036 |
+
|
| 1037 |
+
Returns:
|
| 1038 |
+
:obj:`int`: The number of tokens that were created in the vocabulary
|
| 1039 |
+
"""
|
| 1040 |
+
pass
|
| 1041 |
+
|
| 1042 |
+
def async_decode_batch(self, sequences, skip_special_tokens=True):
|
| 1043 |
+
"""
|
| 1044 |
+
Decode a batch of ids back to their corresponding string
|
| 1045 |
+
|
| 1046 |
+
Args:
|
| 1047 |
+
sequences (:obj:`List` of :obj:`List[int]`):
|
| 1048 |
+
The batch of sequences we want to decode
|
| 1049 |
+
|
| 1050 |
+
skip_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1051 |
+
Whether the special tokens should be removed from the decoded strings
|
| 1052 |
+
|
| 1053 |
+
Returns:
|
| 1054 |
+
:obj:`List[str]`: A list of decoded strings
|
| 1055 |
+
"""
|
| 1056 |
+
pass
|
| 1057 |
+
|
| 1058 |
+
def async_encode(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True):
|
| 1059 |
+
"""
|
| 1060 |
+
Asynchronously encode the given input with character offsets.
|
| 1061 |
+
|
| 1062 |
+
This is an async version of encode that can be awaited in async Python code.
|
| 1063 |
+
|
| 1064 |
+
Example:
|
| 1065 |
+
Here are some examples of the inputs that are accepted::
|
| 1066 |
+
|
| 1067 |
+
await async_encode("A single sequence")
|
| 1068 |
+
|
| 1069 |
+
Args:
|
| 1070 |
+
sequence (:obj:`~tokenizers.InputSequence`):
|
| 1071 |
+
The main input sequence we want to encode. This sequence can be either raw
|
| 1072 |
+
text or pre-tokenized, according to the ``is_pretokenized`` argument:
|
| 1073 |
+
|
| 1074 |
+
- If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence`
|
| 1075 |
+
- If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence`
|
| 1076 |
+
|
| 1077 |
+
pair (:obj:`~tokenizers.InputSequence`, `optional`):
|
| 1078 |
+
An optional input sequence. The expected format is the same that for ``sequence``.
|
| 1079 |
+
|
| 1080 |
+
is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
|
| 1081 |
+
Whether the input is already pre-tokenized
|
| 1082 |
+
|
| 1083 |
+
add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1084 |
+
Whether to add the special tokens
|
| 1085 |
+
|
| 1086 |
+
Returns:
|
| 1087 |
+
:class:`~tokenizers.Encoding`: The encoded result
|
| 1088 |
+
|
| 1089 |
+
"""
|
| 1090 |
+
pass
|
| 1091 |
+
|
| 1092 |
+
def async_encode_batch(self, input, is_pretokenized=False, add_special_tokens=True):
|
| 1093 |
+
"""
|
| 1094 |
+
Asynchronously encode the given batch of inputs with character offsets.
|
| 1095 |
+
|
| 1096 |
+
This is an async version of encode_batch that can be awaited in async Python code.
|
| 1097 |
+
|
| 1098 |
+
Example:
|
| 1099 |
+
Here are some examples of the inputs that are accepted::
|
| 1100 |
+
|
| 1101 |
+
await async_encode_batch([
|
| 1102 |
+
"A single sequence",
|
| 1103 |
+
("A tuple with a sequence", "And its pair"),
|
| 1104 |
+
[ "A", "pre", "tokenized", "sequence" ],
|
| 1105 |
+
([ "A", "pre", "tokenized", "sequence" ], "And its pair")
|
| 1106 |
+
])
|
| 1107 |
+
|
| 1108 |
+
Args:
|
| 1109 |
+
input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
|
| 1110 |
+
A list of single sequences or pair sequences to encode. Each sequence
|
| 1111 |
+
can be either raw text or pre-tokenized, according to the ``is_pretokenized``
|
| 1112 |
+
argument:
|
| 1113 |
+
|
| 1114 |
+
- If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
|
| 1115 |
+
- If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
|
| 1116 |
+
|
| 1117 |
+
is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
|
| 1118 |
+
Whether the input is already pre-tokenized
|
| 1119 |
+
|
| 1120 |
+
add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1121 |
+
Whether to add the special tokens
|
| 1122 |
+
|
| 1123 |
+
Returns:
|
| 1124 |
+
A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
|
| 1125 |
+
|
| 1126 |
+
"""
|
| 1127 |
+
pass
|
| 1128 |
+
|
| 1129 |
+
def async_encode_batch_fast(self, input, is_pretokenized=False, add_special_tokens=True):
|
| 1130 |
+
"""
|
| 1131 |
+
Asynchronously encode the given batch of inputs without tracking character offsets.
|
| 1132 |
+
|
| 1133 |
+
This is an async version of encode_batch_fast that can be awaited in async Python code.
|
| 1134 |
+
|
| 1135 |
+
Example:
|
| 1136 |
+
Here are some examples of the inputs that are accepted::
|
| 1137 |
+
|
| 1138 |
+
await async_encode_batch_fast([
|
| 1139 |
+
"A single sequence",
|
| 1140 |
+
("A tuple with a sequence", "And its pair"),
|
| 1141 |
+
[ "A", "pre", "tokenized", "sequence" ],
|
| 1142 |
+
([ "A", "pre", "tokenized", "sequence" ], "And its pair")
|
| 1143 |
+
])
|
| 1144 |
+
|
| 1145 |
+
Args:
|
| 1146 |
+
input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
|
| 1147 |
+
A list of single sequences or pair sequences to encode. Each sequence
|
| 1148 |
+
can be either raw text or pre-tokenized, according to the ``is_pretokenized``
|
| 1149 |
+
argument:
|
| 1150 |
+
|
| 1151 |
+
- If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
|
| 1152 |
+
- If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
|
| 1153 |
+
|
| 1154 |
+
is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
|
| 1155 |
+
Whether the input is already pre-tokenized
|
| 1156 |
+
|
| 1157 |
+
add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1158 |
+
Whether to add the special tokens
|
| 1159 |
+
|
| 1160 |
+
Returns:
|
| 1161 |
+
A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
|
| 1162 |
+
|
| 1163 |
+
"""
|
| 1164 |
+
pass
|
| 1165 |
+
|
| 1166 |
+
def decode(self, ids, skip_special_tokens=True):
|
| 1167 |
+
"""
|
| 1168 |
+
Decode the given list of ids back to a string
|
| 1169 |
+
|
| 1170 |
+
This is used to decode anything coming back from a Language Model
|
| 1171 |
+
|
| 1172 |
+
Args:
|
| 1173 |
+
ids (A :obj:`List/Tuple` of :obj:`int`):
|
| 1174 |
+
The list of ids that we want to decode
|
| 1175 |
+
|
| 1176 |
+
skip_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1177 |
+
Whether the special tokens should be removed from the decoded string
|
| 1178 |
+
|
| 1179 |
+
Returns:
|
| 1180 |
+
:obj:`str`: The decoded string
|
| 1181 |
+
"""
|
| 1182 |
+
pass
|
| 1183 |
+
|
| 1184 |
+
def decode_batch(self, sequences, skip_special_tokens=True):
|
| 1185 |
+
"""
|
| 1186 |
+
Decode a batch of ids back to their corresponding string
|
| 1187 |
+
|
| 1188 |
+
Args:
|
| 1189 |
+
sequences (:obj:`List` of :obj:`List[int]`):
|
| 1190 |
+
The batch of sequences we want to decode
|
| 1191 |
+
|
| 1192 |
+
skip_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1193 |
+
Whether the special tokens should be removed from the decoded strings
|
| 1194 |
+
|
| 1195 |
+
Returns:
|
| 1196 |
+
:obj:`List[str]`: A list of decoded strings
|
| 1197 |
+
"""
|
| 1198 |
+
pass
|
| 1199 |
+
|
| 1200 |
+
@property
|
| 1201 |
+
def decoder(self):
|
| 1202 |
+
"""
|
| 1203 |
+
The `optional` :class:`~tokenizers.decoders.Decoder` in use by the Tokenizer
|
| 1204 |
+
"""
|
| 1205 |
+
pass
|
| 1206 |
+
|
| 1207 |
+
@decoder.setter
|
| 1208 |
+
def decoder(self, value):
|
| 1209 |
+
"""
|
| 1210 |
+
The `optional` :class:`~tokenizers.decoders.Decoder` in use by the Tokenizer
|
| 1211 |
+
"""
|
| 1212 |
+
pass
|
| 1213 |
+
|
| 1214 |
+
def enable_padding(
|
| 1215 |
+
self, direction="right", pad_id=0, pad_type_id=0, pad_token="[PAD]", length=None, pad_to_multiple_of=None
|
| 1216 |
+
):
|
| 1217 |
+
"""
|
| 1218 |
+
Enable the padding
|
| 1219 |
+
|
| 1220 |
+
Args:
|
| 1221 |
+
direction (:obj:`str`, `optional`, defaults to :obj:`right`):
|
| 1222 |
+
The direction in which to pad. Can be either ``right`` or ``left``
|
| 1223 |
+
|
| 1224 |
+
pad_to_multiple_of (:obj:`int`, `optional`):
|
| 1225 |
+
If specified, the padding length should always snap to the next multiple of the
|
| 1226 |
+
given value. For example if we were going to pad witha length of 250 but
|
| 1227 |
+
``pad_to_multiple_of=8`` then we will pad to 256.
|
| 1228 |
+
|
| 1229 |
+
pad_id (:obj:`int`, defaults to 0):
|
| 1230 |
+
The id to be used when padding
|
| 1231 |
+
|
| 1232 |
+
pad_type_id (:obj:`int`, defaults to 0):
|
| 1233 |
+
The type id to be used when padding
|
| 1234 |
+
|
| 1235 |
+
pad_token (:obj:`str`, defaults to :obj:`[PAD]`):
|
| 1236 |
+
The pad token to be used when padding
|
| 1237 |
+
|
| 1238 |
+
length (:obj:`int`, `optional`):
|
| 1239 |
+
If specified, the length at which to pad. If not specified we pad using the size of
|
| 1240 |
+
the longest sequence in a batch.
|
| 1241 |
+
"""
|
| 1242 |
+
pass
|
| 1243 |
+
|
| 1244 |
+
def enable_truncation(self, max_length, stride=0, strategy="longest_first", direction="right"):
|
| 1245 |
+
"""
|
| 1246 |
+
Enable truncation
|
| 1247 |
+
|
| 1248 |
+
Args:
|
| 1249 |
+
max_length (:obj:`int`):
|
| 1250 |
+
The max length at which to truncate
|
| 1251 |
+
|
| 1252 |
+
stride (:obj:`int`, `optional`):
|
| 1253 |
+
The length of the previous first sequence to be included in the overflowing
|
| 1254 |
+
sequence
|
| 1255 |
+
|
| 1256 |
+
strategy (:obj:`str`, `optional`, defaults to :obj:`longest_first`):
|
| 1257 |
+
The strategy used to truncation. Can be one of ``longest_first``, ``only_first`` or
|
| 1258 |
+
``only_second``.
|
| 1259 |
+
|
| 1260 |
+
direction (:obj:`str`, defaults to :obj:`right`):
|
| 1261 |
+
Truncate direction
|
| 1262 |
+
"""
|
| 1263 |
+
pass
|
| 1264 |
+
|
| 1265 |
+
def encode(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True):
|
| 1266 |
+
"""
|
| 1267 |
+
Encode the given sequence and pair. This method can process raw text sequences
|
| 1268 |
+
as well as already pre-tokenized sequences.
|
| 1269 |
+
|
| 1270 |
+
Example:
|
| 1271 |
+
Here are some examples of the inputs that are accepted::
|
| 1272 |
+
|
| 1273 |
+
encode("A single sequence")`
|
| 1274 |
+
encode("A sequence", "And its pair")`
|
| 1275 |
+
encode([ "A", "pre", "tokenized", "sequence" ], is_pretokenized=True)`
|
| 1276 |
+
encode(
|
| 1277 |
+
[ "A", "pre", "tokenized", "sequence" ], [ "And", "its", "pair" ],
|
| 1278 |
+
is_pretokenized=True
|
| 1279 |
+
)
|
| 1280 |
+
|
| 1281 |
+
Args:
|
| 1282 |
+
sequence (:obj:`~tokenizers.InputSequence`):
|
| 1283 |
+
The main input sequence we want to encode. This sequence can be either raw
|
| 1284 |
+
text or pre-tokenized, according to the ``is_pretokenized`` argument:
|
| 1285 |
+
|
| 1286 |
+
- If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence`
|
| 1287 |
+
- If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence`
|
| 1288 |
+
|
| 1289 |
+
pair (:obj:`~tokenizers.InputSequence`, `optional`):
|
| 1290 |
+
An optional input sequence. The expected format is the same that for ``sequence``.
|
| 1291 |
+
|
| 1292 |
+
is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
|
| 1293 |
+
Whether the input is already pre-tokenized
|
| 1294 |
+
|
| 1295 |
+
add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1296 |
+
Whether to add the special tokens
|
| 1297 |
+
|
| 1298 |
+
Returns:
|
| 1299 |
+
:class:`~tokenizers.Encoding`: The encoded result
|
| 1300 |
+
|
| 1301 |
+
"""
|
| 1302 |
+
pass
|
| 1303 |
+
|
| 1304 |
+
def encode_batch(self, input, is_pretokenized=False, add_special_tokens=True):
|
| 1305 |
+
"""
|
| 1306 |
+
Encode the given batch of inputs. This method accept both raw text sequences
|
| 1307 |
+
as well as already pre-tokenized sequences. The reason we use `PySequence` is
|
| 1308 |
+
because it allows type checking with zero-cost (according to PyO3) as we don't
|
| 1309 |
+
have to convert to check.
|
| 1310 |
+
|
| 1311 |
+
Example:
|
| 1312 |
+
Here are some examples of the inputs that are accepted::
|
| 1313 |
+
|
| 1314 |
+
encode_batch([
|
| 1315 |
+
"A single sequence",
|
| 1316 |
+
("A tuple with a sequence", "And its pair"),
|
| 1317 |
+
[ "A", "pre", "tokenized", "sequence" ],
|
| 1318 |
+
([ "A", "pre", "tokenized", "sequence" ], "And its pair")
|
| 1319 |
+
])
|
| 1320 |
+
|
| 1321 |
+
Args:
|
| 1322 |
+
input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
|
| 1323 |
+
A list of single sequences or pair sequences to encode. Each sequence
|
| 1324 |
+
can be either raw text or pre-tokenized, according to the ``is_pretokenized``
|
| 1325 |
+
argument:
|
| 1326 |
+
|
| 1327 |
+
- If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
|
| 1328 |
+
- If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
|
| 1329 |
+
|
| 1330 |
+
is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
|
| 1331 |
+
Whether the input is already pre-tokenized
|
| 1332 |
+
|
| 1333 |
+
add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1334 |
+
Whether to add the special tokens
|
| 1335 |
+
|
| 1336 |
+
Returns:
|
| 1337 |
+
A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
|
| 1338 |
+
|
| 1339 |
+
"""
|
| 1340 |
+
pass
|
| 1341 |
+
|
| 1342 |
+
def encode_batch_fast(self, input, is_pretokenized=False, add_special_tokens=True):
|
| 1343 |
+
"""
|
| 1344 |
+
Encode the given batch of inputs. This method is faster than `encode_batch`
|
| 1345 |
+
because it doesn't keep track of offsets, they will be all zeros.
|
| 1346 |
+
|
| 1347 |
+
Example:
|
| 1348 |
+
Here are some examples of the inputs that are accepted::
|
| 1349 |
+
|
| 1350 |
+
encode_batch_fast([
|
| 1351 |
+
"A single sequence",
|
| 1352 |
+
("A tuple with a sequence", "And its pair"),
|
| 1353 |
+
[ "A", "pre", "tokenized", "sequence" ],
|
| 1354 |
+
([ "A", "pre", "tokenized", "sequence" ], "And its pair")
|
| 1355 |
+
])
|
| 1356 |
+
|
| 1357 |
+
Args:
|
| 1358 |
+
input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
|
| 1359 |
+
A list of single sequences or pair sequences to encode. Each sequence
|
| 1360 |
+
can be either raw text or pre-tokenized, according to the ``is_pretokenized``
|
| 1361 |
+
argument:
|
| 1362 |
+
|
| 1363 |
+
- If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
|
| 1364 |
+
- If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
|
| 1365 |
+
|
| 1366 |
+
is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
|
| 1367 |
+
Whether the input is already pre-tokenized
|
| 1368 |
+
|
| 1369 |
+
add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1370 |
+
Whether to add the special tokens
|
| 1371 |
+
|
| 1372 |
+
Returns:
|
| 1373 |
+
A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
|
| 1374 |
+
|
| 1375 |
+
"""
|
| 1376 |
+
pass
|
| 1377 |
+
|
| 1378 |
+
@property
|
| 1379 |
+
def encode_special_tokens(self):
|
| 1380 |
+
"""
|
| 1381 |
+
Modifies the tokenizer in order to use or not the special tokens
|
| 1382 |
+
during encoding.
|
| 1383 |
+
|
| 1384 |
+
Args:
|
| 1385 |
+
value (:obj:`bool`):
|
| 1386 |
+
Whether to use the special tokens or not
|
| 1387 |
+
|
| 1388 |
+
"""
|
| 1389 |
+
pass
|
| 1390 |
+
|
| 1391 |
+
@encode_special_tokens.setter
|
| 1392 |
+
def encode_special_tokens(self, value):
|
| 1393 |
+
"""
|
| 1394 |
+
Modifies the tokenizer in order to use or not the special tokens
|
| 1395 |
+
during encoding.
|
| 1396 |
+
|
| 1397 |
+
Args:
|
| 1398 |
+
value (:obj:`bool`):
|
| 1399 |
+
Whether to use the special tokens or not
|
| 1400 |
+
|
| 1401 |
+
"""
|
| 1402 |
+
pass
|
| 1403 |
+
|
| 1404 |
+
@staticmethod
|
| 1405 |
+
def from_buffer(buffer):
|
| 1406 |
+
"""
|
| 1407 |
+
Instantiate a new :class:`~tokenizers.Tokenizer` from the given buffer.
|
| 1408 |
+
|
| 1409 |
+
Args:
|
| 1410 |
+
buffer (:obj:`bytes`):
|
| 1411 |
+
A buffer containing a previously serialized :class:`~tokenizers.Tokenizer`
|
| 1412 |
+
|
| 1413 |
+
Returns:
|
| 1414 |
+
:class:`~tokenizers.Tokenizer`: The new tokenizer
|
| 1415 |
+
"""
|
| 1416 |
+
pass
|
| 1417 |
+
|
| 1418 |
+
@staticmethod
|
| 1419 |
+
def from_file(path):
|
| 1420 |
+
"""
|
| 1421 |
+
Instantiate a new :class:`~tokenizers.Tokenizer` from the file at the given path.
|
| 1422 |
+
|
| 1423 |
+
Args:
|
| 1424 |
+
path (:obj:`str`):
|
| 1425 |
+
A path to a local JSON file representing a previously serialized
|
| 1426 |
+
:class:`~tokenizers.Tokenizer`
|
| 1427 |
+
|
| 1428 |
+
Returns:
|
| 1429 |
+
:class:`~tokenizers.Tokenizer`: The new tokenizer
|
| 1430 |
+
"""
|
| 1431 |
+
pass
|
| 1432 |
+
|
| 1433 |
+
@staticmethod
|
| 1434 |
+
def from_pretrained(identifier, revision="main", token=None):
|
| 1435 |
+
"""
|
| 1436 |
+
Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
|
| 1437 |
+
Hugging Face Hub.
|
| 1438 |
+
|
| 1439 |
+
Args:
|
| 1440 |
+
identifier (:obj:`str`):
|
| 1441 |
+
The identifier of a Model on the Hugging Face Hub, that contains
|
| 1442 |
+
a tokenizer.json file
|
| 1443 |
+
revision (:obj:`str`, defaults to `main`):
|
| 1444 |
+
A branch or commit id
|
| 1445 |
+
token (:obj:`str`, `optional`, defaults to `None`):
|
| 1446 |
+
An optional auth token used to access private repositories on the
|
| 1447 |
+
Hugging Face Hub
|
| 1448 |
+
|
| 1449 |
+
Returns:
|
| 1450 |
+
:class:`~tokenizers.Tokenizer`: The new tokenizer
|
| 1451 |
+
"""
|
| 1452 |
+
pass
|
| 1453 |
+
|
| 1454 |
+
@staticmethod
|
| 1455 |
+
def from_str(json):
|
| 1456 |
+
"""
|
| 1457 |
+
Instantiate a new :class:`~tokenizers.Tokenizer` from the given JSON string.
|
| 1458 |
+
|
| 1459 |
+
Args:
|
| 1460 |
+
json (:obj:`str`):
|
| 1461 |
+
A valid JSON string representing a previously serialized
|
| 1462 |
+
:class:`~tokenizers.Tokenizer`
|
| 1463 |
+
|
| 1464 |
+
Returns:
|
| 1465 |
+
:class:`~tokenizers.Tokenizer`: The new tokenizer
|
| 1466 |
+
"""
|
| 1467 |
+
pass
|
| 1468 |
+
|
| 1469 |
+
def get_added_tokens_decoder(self):
|
| 1470 |
+
"""
|
| 1471 |
+
Get the underlying vocabulary
|
| 1472 |
+
|
| 1473 |
+
Returns:
|
| 1474 |
+
:obj:`Dict[int, AddedToken]`: The vocabulary
|
| 1475 |
+
"""
|
| 1476 |
+
pass
|
| 1477 |
+
|
| 1478 |
+
def get_vocab(self, with_added_tokens=True):
|
| 1479 |
+
"""
|
| 1480 |
+
Get the underlying vocabulary
|
| 1481 |
+
|
| 1482 |
+
Args:
|
| 1483 |
+
with_added_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1484 |
+
Whether to include the added tokens
|
| 1485 |
+
|
| 1486 |
+
Returns:
|
| 1487 |
+
:obj:`Dict[str, int]`: The vocabulary
|
| 1488 |
+
"""
|
| 1489 |
+
pass
|
| 1490 |
+
|
| 1491 |
+
def get_vocab_size(self, with_added_tokens=True):
|
| 1492 |
+
"""
|
| 1493 |
+
Get the size of the underlying vocabulary
|
| 1494 |
+
|
| 1495 |
+
Args:
|
| 1496 |
+
with_added_tokens (:obj:`bool`, defaults to :obj:`True`):
|
| 1497 |
+
Whether to include the added tokens
|
| 1498 |
+
|
| 1499 |
+
Returns:
|
| 1500 |
+
:obj:`int`: The size of the vocabulary
|
| 1501 |
+
"""
|
| 1502 |
+
pass
|
| 1503 |
+
|
| 1504 |
+
def id_to_token(self, id):
|
| 1505 |
+
"""
|
| 1506 |
+
Convert the given id to its corresponding token if it exists
|
| 1507 |
+
|
| 1508 |
+
Args:
|
| 1509 |
+
id (:obj:`int`):
|
| 1510 |
+
The id to convert
|
| 1511 |
+
|
| 1512 |
+
Returns:
|
| 1513 |
+
:obj:`Optional[str]`: An optional token, :obj:`None` if out of vocabulary
|
| 1514 |
+
"""
|
| 1515 |
+
pass
|
| 1516 |
+
|
| 1517 |
+
@property
|
| 1518 |
+
def model(self):
|
| 1519 |
+
"""
|
| 1520 |
+
The :class:`~tokenizers.models.Model` in use by the Tokenizer
|
| 1521 |
+
"""
|
| 1522 |
+
pass
|
| 1523 |
+
|
| 1524 |
+
@model.setter
|
| 1525 |
+
def model(self, value):
|
| 1526 |
+
"""
|
| 1527 |
+
The :class:`~tokenizers.models.Model` in use by the Tokenizer
|
| 1528 |
+
"""
|
| 1529 |
+
pass
|
| 1530 |
+
|
| 1531 |
+
def no_padding(self):
|
| 1532 |
+
"""
|
| 1533 |
+
Disable padding
|
| 1534 |
+
"""
|
| 1535 |
+
pass
|
| 1536 |
+
|
| 1537 |
+
def no_truncation(self):
|
| 1538 |
+
"""
|
| 1539 |
+
Disable truncation
|
| 1540 |
+
"""
|
| 1541 |
+
pass
|
| 1542 |
+
|
| 1543 |
+
@property
|
| 1544 |
+
def normalizer(self):
|
| 1545 |
+
"""
|
| 1546 |
+
The `optional` :class:`~tokenizers.normalizers.Normalizer` in use by the Tokenizer
|
| 1547 |
+
"""
|
| 1548 |
+
pass
|
| 1549 |
+
|
| 1550 |
+
@normalizer.setter
|
| 1551 |
+
def normalizer(self, value):
|
| 1552 |
+
"""
|
| 1553 |
+
The `optional` :class:`~tokenizers.normalizers.Normalizer` in use by the Tokenizer
|
| 1554 |
+
"""
|
| 1555 |
+
pass
|
| 1556 |
+
|
| 1557 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 1558 |
+
"""
|
| 1559 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 1560 |
+
:param is_pair: Boolean indicating if the input would be a single sentence or a pair
|
| 1561 |
+
:return:
|
| 1562 |
+
"""
|
| 1563 |
+
pass
|
| 1564 |
+
|
| 1565 |
+
@property
|
| 1566 |
+
def padding(self):
|
| 1567 |
+
"""
|
| 1568 |
+
Get the current padding parameters
|
| 1569 |
+
|
| 1570 |
+
`Cannot be set, use` :meth:`~tokenizers.Tokenizer.enable_padding` `instead`
|
| 1571 |
+
|
| 1572 |
+
Returns:
|
| 1573 |
+
(:obj:`dict`, `optional`):
|
| 1574 |
+
A dict with the current padding parameters if padding is enabled
|
| 1575 |
+
"""
|
| 1576 |
+
pass
|
| 1577 |
+
|
| 1578 |
+
@padding.setter
|
| 1579 |
+
def padding(self, value):
|
| 1580 |
+
"""
|
| 1581 |
+
Get the current padding parameters
|
| 1582 |
+
|
| 1583 |
+
`Cannot be set, use` :meth:`~tokenizers.Tokenizer.enable_padding` `instead`
|
| 1584 |
+
|
| 1585 |
+
Returns:
|
| 1586 |
+
(:obj:`dict`, `optional`):
|
| 1587 |
+
A dict with the current padding parameters if padding is enabled
|
| 1588 |
+
"""
|
| 1589 |
+
pass
|
| 1590 |
+
|
| 1591 |
+
def post_process(self, encoding, pair=None, add_special_tokens=True):
|
| 1592 |
+
"""
|
| 1593 |
+
Apply all the post-processing steps to the given encodings.
|
| 1594 |
+
|
| 1595 |
+
The various steps are:
|
| 1596 |
+
|
| 1597 |
+
1. Truncate according to the set truncation params (provided with
|
| 1598 |
+
:meth:`~tokenizers.Tokenizer.enable_truncation`)
|
| 1599 |
+
2. Apply the :class:`~tokenizers.processors.PostProcessor`
|
| 1600 |
+
3. Pad according to the set padding params (provided with
|
| 1601 |
+
:meth:`~tokenizers.Tokenizer.enable_padding`)
|
| 1602 |
+
|
| 1603 |
+
Args:
|
| 1604 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 1605 |
+
The :class:`~tokenizers.Encoding` corresponding to the main sequence.
|
| 1606 |
+
|
| 1607 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 1608 |
+
An optional :class:`~tokenizers.Encoding` corresponding to the pair sequence.
|
| 1609 |
+
|
| 1610 |
+
add_special_tokens (:obj:`bool`):
|
| 1611 |
+
Whether to add the special tokens
|
| 1612 |
+
|
| 1613 |
+
Returns:
|
| 1614 |
+
:class:`~tokenizers.Encoding`: The final post-processed encoding
|
| 1615 |
+
"""
|
| 1616 |
+
pass
|
| 1617 |
+
|
| 1618 |
+
@property
|
| 1619 |
+
def post_processor(self):
|
| 1620 |
+
"""
|
| 1621 |
+
The `optional` :class:`~tokenizers.processors.PostProcessor` in use by the Tokenizer
|
| 1622 |
+
"""
|
| 1623 |
+
pass
|
| 1624 |
+
|
| 1625 |
+
@post_processor.setter
|
| 1626 |
+
def post_processor(self, value):
|
| 1627 |
+
"""
|
| 1628 |
+
The `optional` :class:`~tokenizers.processors.PostProcessor` in use by the Tokenizer
|
| 1629 |
+
"""
|
| 1630 |
+
pass
|
| 1631 |
+
|
| 1632 |
+
@property
|
| 1633 |
+
def pre_tokenizer(self):
|
| 1634 |
+
"""
|
| 1635 |
+
The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer
|
| 1636 |
+
"""
|
| 1637 |
+
pass
|
| 1638 |
+
|
| 1639 |
+
@pre_tokenizer.setter
|
| 1640 |
+
def pre_tokenizer(self, value):
|
| 1641 |
+
"""
|
| 1642 |
+
The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer
|
| 1643 |
+
"""
|
| 1644 |
+
pass
|
| 1645 |
+
|
| 1646 |
+
def save(self, path, pretty=True):
|
| 1647 |
+
"""
|
| 1648 |
+
Save the :class:`~tokenizers.Tokenizer` to the file at the given path.
|
| 1649 |
+
|
| 1650 |
+
Args:
|
| 1651 |
+
path (:obj:`str`):
|
| 1652 |
+
A path to a file in which to save the serialized tokenizer.
|
| 1653 |
+
|
| 1654 |
+
pretty (:obj:`bool`, defaults to :obj:`True`):
|
| 1655 |
+
Whether the JSON file should be pretty formatted.
|
| 1656 |
+
"""
|
| 1657 |
+
pass
|
| 1658 |
+
|
| 1659 |
+
def to_str(self, pretty=False):
|
| 1660 |
+
"""
|
| 1661 |
+
Gets a serialized string representing this :class:`~tokenizers.Tokenizer`.
|
| 1662 |
+
|
| 1663 |
+
Args:
|
| 1664 |
+
pretty (:obj:`bool`, defaults to :obj:`False`):
|
| 1665 |
+
Whether the JSON string should be pretty formatted.
|
| 1666 |
+
|
| 1667 |
+
Returns:
|
| 1668 |
+
:obj:`str`: A string representing the serialized Tokenizer
|
| 1669 |
+
"""
|
| 1670 |
+
pass
|
| 1671 |
+
|
| 1672 |
+
def token_to_id(self, token):
|
| 1673 |
+
"""
|
| 1674 |
+
Convert the given token to its corresponding id if it exists
|
| 1675 |
+
|
| 1676 |
+
Args:
|
| 1677 |
+
token (:obj:`str`):
|
| 1678 |
+
The token to convert
|
| 1679 |
+
|
| 1680 |
+
Returns:
|
| 1681 |
+
:obj:`Optional[int]`: An optional id, :obj:`None` if out of vocabulary
|
| 1682 |
+
"""
|
| 1683 |
+
pass
|
| 1684 |
+
|
| 1685 |
+
def train(self, files, trainer=None):
|
| 1686 |
+
"""
|
| 1687 |
+
Train the Tokenizer using the given files.
|
| 1688 |
+
|
| 1689 |
+
Reads the files line by line, while keeping all the whitespace, even new lines.
|
| 1690 |
+
If you want to train from data store in-memory, you can check
|
| 1691 |
+
:meth:`~tokenizers.Tokenizer.train_from_iterator`
|
| 1692 |
+
|
| 1693 |
+
Args:
|
| 1694 |
+
files (:obj:`List[str]`):
|
| 1695 |
+
A list of path to the files that we should use for training
|
| 1696 |
+
|
| 1697 |
+
trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`):
|
| 1698 |
+
An optional trainer that should be used to train our Model
|
| 1699 |
+
"""
|
| 1700 |
+
pass
|
| 1701 |
+
|
| 1702 |
+
def train_from_iterator(self, iterator, trainer=None, length=None):
|
| 1703 |
+
"""
|
| 1704 |
+
Train the Tokenizer using the provided iterator.
|
| 1705 |
+
|
| 1706 |
+
You can provide anything that is a Python Iterator
|
| 1707 |
+
|
| 1708 |
+
* A list of sequences :obj:`List[str]`
|
| 1709 |
+
* A generator that yields :obj:`str` or :obj:`List[str]`
|
| 1710 |
+
* A Numpy array of strings
|
| 1711 |
+
* ...
|
| 1712 |
+
|
| 1713 |
+
Args:
|
| 1714 |
+
iterator (:obj:`Iterator`):
|
| 1715 |
+
Any iterator over strings or list of strings
|
| 1716 |
+
|
| 1717 |
+
trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`):
|
| 1718 |
+
An optional trainer that should be used to train our Model
|
| 1719 |
+
|
| 1720 |
+
length (:obj:`int`, `optional`):
|
| 1721 |
+
The total number of sequences in the iterator. This is used to
|
| 1722 |
+
provide meaningful progress tracking
|
| 1723 |
+
"""
|
| 1724 |
+
pass
|
| 1725 |
+
|
| 1726 |
+
@property
|
| 1727 |
+
def truncation(self):
|
| 1728 |
+
"""
|
| 1729 |
+
Get the currently set truncation parameters
|
| 1730 |
+
|
| 1731 |
+
`Cannot set, use` :meth:`~tokenizers.Tokenizer.enable_truncation` `instead`
|
| 1732 |
+
|
| 1733 |
+
Returns:
|
| 1734 |
+
(:obj:`dict`, `optional`):
|
| 1735 |
+
A dict with the current truncation parameters if truncation is enabled
|
| 1736 |
+
"""
|
| 1737 |
+
pass
|
| 1738 |
+
|
| 1739 |
+
@truncation.setter
|
| 1740 |
+
def truncation(self, value):
|
| 1741 |
+
"""
|
| 1742 |
+
Get the currently set truncation parameters
|
| 1743 |
+
|
| 1744 |
+
`Cannot set, use` :meth:`~tokenizers.Tokenizer.enable_truncation` `instead`
|
| 1745 |
+
|
| 1746 |
+
Returns:
|
| 1747 |
+
(:obj:`dict`, `optional`):
|
| 1748 |
+
A dict with the current truncation parameters if truncation is enabled
|
| 1749 |
+
"""
|
| 1750 |
+
pass
|
| 1751 |
+
|
| 1752 |
+
from enum import Enum
|
| 1753 |
+
from typing import List, Tuple, Union, Any
|
| 1754 |
+
|
| 1755 |
+
Offsets = Tuple[int, int]
|
| 1756 |
+
TextInputSequence = str
|
| 1757 |
+
PreTokenizedInputSequence = Union[List[str], Tuple[str, ...]]
|
| 1758 |
+
TextEncodeInput = Union[
|
| 1759 |
+
TextInputSequence,
|
| 1760 |
+
Tuple[TextInputSequence, TextInputSequence],
|
| 1761 |
+
List[TextInputSequence],
|
| 1762 |
+
]
|
| 1763 |
+
PreTokenizedEncodeInput = Union[
|
| 1764 |
+
PreTokenizedInputSequence,
|
| 1765 |
+
Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
|
| 1766 |
+
List[PreTokenizedInputSequence],
|
| 1767 |
+
]
|
| 1768 |
+
InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
|
| 1769 |
+
EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
|
| 1770 |
+
|
| 1771 |
+
class OffsetReferential(Enum):
|
| 1772 |
+
ORIGINAL = "original"
|
| 1773 |
+
NORMALIZED = "normalized"
|
| 1774 |
+
|
| 1775 |
+
class OffsetType(Enum):
|
| 1776 |
+
BYTE = "byte"
|
| 1777 |
+
CHAR = "char"
|
| 1778 |
+
|
| 1779 |
+
class SplitDelimiterBehavior(Enum):
|
| 1780 |
+
REMOVED = "removed"
|
| 1781 |
+
ISOLATED = "isolated"
|
| 1782 |
+
MERGED_WITH_PREVIOUS = "merged_with_previous"
|
| 1783 |
+
MERGED_WITH_NEXT = "merged_with_next"
|
| 1784 |
+
CONTIGUOUS = "contiguous"
|
| 1785 |
+
|
| 1786 |
+
from .implementations import (
|
| 1787 |
+
BertWordPieceTokenizer,
|
| 1788 |
+
ByteLevelBPETokenizer,
|
| 1789 |
+
CharBPETokenizer,
|
| 1790 |
+
SentencePieceBPETokenizer,
|
| 1791 |
+
SentencePieceUnigramTokenizer,
|
| 1792 |
+
)
|
| 1793 |
+
|
| 1794 |
+
def __getattr__(name: str) -> Any: ...
|
| 1795 |
+
|
| 1796 |
+
BertWordPieceTokenizer: Any
|
| 1797 |
+
ByteLevelBPETokenizer: Any
|
| 1798 |
+
CharBPETokenizer: Any
|
| 1799 |
+
SentencePieceBPETokenizer: Any
|
| 1800 |
+
SentencePieceUnigramTokenizer: Any
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/decoders/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .. import decoders
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Decoder = decoders.Decoder
|
| 5 |
+
ByteLevel = decoders.ByteLevel
|
| 6 |
+
Replace = decoders.Replace
|
| 7 |
+
WordPiece = decoders.WordPiece
|
| 8 |
+
ByteFallback = decoders.ByteFallback
|
| 9 |
+
Fuse = decoders.Fuse
|
| 10 |
+
Strip = decoders.Strip
|
| 11 |
+
Metaspace = decoders.Metaspace
|
| 12 |
+
BPEDecoder = decoders.BPEDecoder
|
| 13 |
+
CTC = decoders.CTC
|
| 14 |
+
Sequence = decoders.Sequence
|
| 15 |
+
DecodeStream = decoders.DecodeStream
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/decoders/__init__.pyi
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class DecodeStream:
|
| 3 |
+
"""
|
| 4 |
+
Class needed for streaming decode
|
| 5 |
+
|
| 6 |
+
"""
|
| 7 |
+
def __init__(self, ids=None, skip_special_tokens=False):
|
| 8 |
+
pass
|
| 9 |
+
|
| 10 |
+
def __getstate__(self, /):
|
| 11 |
+
"""
|
| 12 |
+
Helper for pickle.
|
| 13 |
+
"""
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
def step(self, tokenizer, id):
|
| 17 |
+
"""
|
| 18 |
+
Streaming decode step
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
tokenizer (:class:`~tokenizers.Tokenizer`):
|
| 22 |
+
The tokenizer to use for decoding
|
| 23 |
+
id (:obj:`int` or `List[int]`):
|
| 24 |
+
The next token id or list of token ids to add to the stream
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
Returns:
|
| 28 |
+
:obj:`Optional[str]`: The next decoded string chunk, or None if not enough
|
| 29 |
+
tokens have been provided yet.
|
| 30 |
+
"""
|
| 31 |
+
pass
|
| 32 |
+
|
| 33 |
+
class Decoder:
|
| 34 |
+
"""
|
| 35 |
+
Base class for all decoders
|
| 36 |
+
|
| 37 |
+
This class is not supposed to be instantiated directly. Instead, any implementation of
|
| 38 |
+
a Decoder will return an instance of this class when instantiated.
|
| 39 |
+
"""
|
| 40 |
+
def __getstate__(self):
|
| 41 |
+
""" """
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
def __setstate__(self, state):
|
| 45 |
+
""" """
|
| 46 |
+
pass
|
| 47 |
+
|
| 48 |
+
@staticmethod
|
| 49 |
+
def custom(decoder):
|
| 50 |
+
""" """
|
| 51 |
+
pass
|
| 52 |
+
|
| 53 |
+
def decode(self, tokens):
|
| 54 |
+
"""
|
| 55 |
+
Decode the given list of tokens to a final string
|
| 56 |
+
|
| 57 |
+
Args:
|
| 58 |
+
tokens (:obj:`List[str]`):
|
| 59 |
+
The list of tokens to decode
|
| 60 |
+
|
| 61 |
+
Returns:
|
| 62 |
+
:obj:`str`: The decoded string
|
| 63 |
+
"""
|
| 64 |
+
pass
|
| 65 |
+
|
| 66 |
+
class BPEDecoder(Decoder):
|
| 67 |
+
"""
|
| 68 |
+
BPEDecoder Decoder
|
| 69 |
+
|
| 70 |
+
Args:
|
| 71 |
+
suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
|
| 72 |
+
The suffix that was used to characterize an end-of-word. This suffix will
|
| 73 |
+
be replaced by whitespaces during the decoding
|
| 74 |
+
"""
|
| 75 |
+
def __init__(self, suffix="</w>"):
|
| 76 |
+
pass
|
| 77 |
+
|
| 78 |
+
def __getstate__(self):
|
| 79 |
+
""" """
|
| 80 |
+
pass
|
| 81 |
+
|
| 82 |
+
def __setstate__(self, state):
|
| 83 |
+
""" """
|
| 84 |
+
pass
|
| 85 |
+
|
| 86 |
+
@staticmethod
|
| 87 |
+
def custom(decoder):
|
| 88 |
+
""" """
|
| 89 |
+
pass
|
| 90 |
+
|
| 91 |
+
def decode(self, tokens):
|
| 92 |
+
"""
|
| 93 |
+
Decode the given list of tokens to a final string
|
| 94 |
+
|
| 95 |
+
Args:
|
| 96 |
+
tokens (:obj:`List[str]`):
|
| 97 |
+
The list of tokens to decode
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
:obj:`str`: The decoded string
|
| 101 |
+
"""
|
| 102 |
+
pass
|
| 103 |
+
|
| 104 |
+
@property
|
| 105 |
+
def suffix(self):
|
| 106 |
+
""" """
|
| 107 |
+
pass
|
| 108 |
+
|
| 109 |
+
@suffix.setter
|
| 110 |
+
def suffix(self, value):
|
| 111 |
+
""" """
|
| 112 |
+
pass
|
| 113 |
+
|
| 114 |
+
class ByteFallback(Decoder):
|
| 115 |
+
"""
|
| 116 |
+
ByteFallback Decoder
|
| 117 |
+
ByteFallback is a simple trick which converts tokens looking like `<0x61>`
|
| 118 |
+
to pure bytes, and attempts to make them into a string. If the tokens
|
| 119 |
+
cannot be decoded you will get � instead for each inconvertible byte token
|
| 120 |
+
|
| 121 |
+
"""
|
| 122 |
+
def __init__(self):
|
| 123 |
+
pass
|
| 124 |
+
|
| 125 |
+
def __getstate__(self):
|
| 126 |
+
""" """
|
| 127 |
+
pass
|
| 128 |
+
|
| 129 |
+
def __setstate__(self, state):
|
| 130 |
+
""" """
|
| 131 |
+
pass
|
| 132 |
+
|
| 133 |
+
@staticmethod
|
| 134 |
+
def custom(decoder):
|
| 135 |
+
""" """
|
| 136 |
+
pass
|
| 137 |
+
|
| 138 |
+
def decode(self, tokens):
|
| 139 |
+
"""
|
| 140 |
+
Decode the given list of tokens to a final string
|
| 141 |
+
|
| 142 |
+
Args:
|
| 143 |
+
tokens (:obj:`List[str]`):
|
| 144 |
+
The list of tokens to decode
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
:obj:`str`: The decoded string
|
| 148 |
+
"""
|
| 149 |
+
pass
|
| 150 |
+
|
| 151 |
+
class ByteLevel(Decoder):
|
| 152 |
+
"""
|
| 153 |
+
ByteLevel Decoder
|
| 154 |
+
|
| 155 |
+
This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.ByteLevel`
|
| 156 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`.
|
| 157 |
+
"""
|
| 158 |
+
def __init__(self):
|
| 159 |
+
pass
|
| 160 |
+
|
| 161 |
+
def __getstate__(self):
|
| 162 |
+
""" """
|
| 163 |
+
pass
|
| 164 |
+
|
| 165 |
+
def __setstate__(self, state):
|
| 166 |
+
""" """
|
| 167 |
+
pass
|
| 168 |
+
|
| 169 |
+
@staticmethod
|
| 170 |
+
def custom(decoder):
|
| 171 |
+
""" """
|
| 172 |
+
pass
|
| 173 |
+
|
| 174 |
+
def decode(self, tokens):
|
| 175 |
+
"""
|
| 176 |
+
Decode the given list of tokens to a final string
|
| 177 |
+
|
| 178 |
+
Args:
|
| 179 |
+
tokens (:obj:`List[str]`):
|
| 180 |
+
The list of tokens to decode
|
| 181 |
+
|
| 182 |
+
Returns:
|
| 183 |
+
:obj:`str`: The decoded string
|
| 184 |
+
"""
|
| 185 |
+
pass
|
| 186 |
+
|
| 187 |
+
class CTC(Decoder):
|
| 188 |
+
"""
|
| 189 |
+
CTC Decoder
|
| 190 |
+
|
| 191 |
+
Args:
|
| 192 |
+
pad_token (:obj:`str`, `optional`, defaults to :obj:`<pad>`):
|
| 193 |
+
The pad token used by CTC to delimit a new token.
|
| 194 |
+
word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`):
|
| 195 |
+
The word delimiter token. It will be replaced by a <space>
|
| 196 |
+
cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 197 |
+
Whether to cleanup some tokenization artifacts.
|
| 198 |
+
Mainly spaces before punctuation, and some abbreviated english forms.
|
| 199 |
+
"""
|
| 200 |
+
def __init__(self, pad_token="<pad>", word_delimiter_token="|", cleanup=True):
|
| 201 |
+
pass
|
| 202 |
+
|
| 203 |
+
def __getstate__(self):
|
| 204 |
+
""" """
|
| 205 |
+
pass
|
| 206 |
+
|
| 207 |
+
def __setstate__(self, state):
|
| 208 |
+
""" """
|
| 209 |
+
pass
|
| 210 |
+
|
| 211 |
+
@property
|
| 212 |
+
def cleanup(self):
|
| 213 |
+
""" """
|
| 214 |
+
pass
|
| 215 |
+
|
| 216 |
+
@cleanup.setter
|
| 217 |
+
def cleanup(self, value):
|
| 218 |
+
""" """
|
| 219 |
+
pass
|
| 220 |
+
|
| 221 |
+
@staticmethod
|
| 222 |
+
def custom(decoder):
|
| 223 |
+
""" """
|
| 224 |
+
pass
|
| 225 |
+
|
| 226 |
+
def decode(self, tokens):
|
| 227 |
+
"""
|
| 228 |
+
Decode the given list of tokens to a final string
|
| 229 |
+
|
| 230 |
+
Args:
|
| 231 |
+
tokens (:obj:`List[str]`):
|
| 232 |
+
The list of tokens to decode
|
| 233 |
+
|
| 234 |
+
Returns:
|
| 235 |
+
:obj:`str`: The decoded string
|
| 236 |
+
"""
|
| 237 |
+
pass
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def pad_token(self):
|
| 241 |
+
""" """
|
| 242 |
+
pass
|
| 243 |
+
|
| 244 |
+
@pad_token.setter
|
| 245 |
+
def pad_token(self, value):
|
| 246 |
+
""" """
|
| 247 |
+
pass
|
| 248 |
+
|
| 249 |
+
@property
|
| 250 |
+
def word_delimiter_token(self):
|
| 251 |
+
""" """
|
| 252 |
+
pass
|
| 253 |
+
|
| 254 |
+
@word_delimiter_token.setter
|
| 255 |
+
def word_delimiter_token(self, value):
|
| 256 |
+
""" """
|
| 257 |
+
pass
|
| 258 |
+
|
| 259 |
+
class Fuse(Decoder):
|
| 260 |
+
"""
|
| 261 |
+
Fuse Decoder
|
| 262 |
+
Fuse simply fuses every token into a single string.
|
| 263 |
+
This is the last step of decoding, this decoder exists only if
|
| 264 |
+
there is need to add other decoders *after* the fusion
|
| 265 |
+
"""
|
| 266 |
+
def __init__(self):
|
| 267 |
+
pass
|
| 268 |
+
|
| 269 |
+
def __getstate__(self):
|
| 270 |
+
""" """
|
| 271 |
+
pass
|
| 272 |
+
|
| 273 |
+
def __setstate__(self, state):
|
| 274 |
+
""" """
|
| 275 |
+
pass
|
| 276 |
+
|
| 277 |
+
@staticmethod
|
| 278 |
+
def custom(decoder):
|
| 279 |
+
""" """
|
| 280 |
+
pass
|
| 281 |
+
|
| 282 |
+
def decode(self, tokens):
|
| 283 |
+
"""
|
| 284 |
+
Decode the given list of tokens to a final string
|
| 285 |
+
|
| 286 |
+
Args:
|
| 287 |
+
tokens (:obj:`List[str]`):
|
| 288 |
+
The list of tokens to decode
|
| 289 |
+
|
| 290 |
+
Returns:
|
| 291 |
+
:obj:`str`: The decoded string
|
| 292 |
+
"""
|
| 293 |
+
pass
|
| 294 |
+
|
| 295 |
+
class Metaspace(Decoder):
|
| 296 |
+
"""
|
| 297 |
+
Metaspace Decoder
|
| 298 |
+
|
| 299 |
+
Args:
|
| 300 |
+
replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
|
| 301 |
+
The replacement character. Must be exactly one character. By default we
|
| 302 |
+
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
|
| 303 |
+
|
| 304 |
+
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
|
| 305 |
+
Whether to add a space to the first word if there isn't already one. This
|
| 306 |
+
lets us treat `hello` exactly like `say hello`.
|
| 307 |
+
Choices: "always", "never", "first". First means the space is only added on the first
|
| 308 |
+
token (relevant when special tokens are used or other pre_tokenizer are used).
|
| 309 |
+
"""
|
| 310 |
+
def __init__(self, replacement="▁", prepend_scheme="always", split=True):
|
| 311 |
+
pass
|
| 312 |
+
|
| 313 |
+
def __getstate__(self):
|
| 314 |
+
""" """
|
| 315 |
+
pass
|
| 316 |
+
|
| 317 |
+
def __setstate__(self, state):
|
| 318 |
+
""" """
|
| 319 |
+
pass
|
| 320 |
+
|
| 321 |
+
@staticmethod
|
| 322 |
+
def custom(decoder):
|
| 323 |
+
""" """
|
| 324 |
+
pass
|
| 325 |
+
|
| 326 |
+
def decode(self, tokens):
|
| 327 |
+
"""
|
| 328 |
+
Decode the given list of tokens to a final string
|
| 329 |
+
|
| 330 |
+
Args:
|
| 331 |
+
tokens (:obj:`List[str]`):
|
| 332 |
+
The list of tokens to decode
|
| 333 |
+
|
| 334 |
+
Returns:
|
| 335 |
+
:obj:`str`: The decoded string
|
| 336 |
+
"""
|
| 337 |
+
pass
|
| 338 |
+
|
| 339 |
+
@property
|
| 340 |
+
def prepend_scheme(self):
|
| 341 |
+
""" """
|
| 342 |
+
pass
|
| 343 |
+
|
| 344 |
+
@prepend_scheme.setter
|
| 345 |
+
def prepend_scheme(self, value):
|
| 346 |
+
""" """
|
| 347 |
+
pass
|
| 348 |
+
|
| 349 |
+
@property
|
| 350 |
+
def replacement(self):
|
| 351 |
+
""" """
|
| 352 |
+
pass
|
| 353 |
+
|
| 354 |
+
@replacement.setter
|
| 355 |
+
def replacement(self, value):
|
| 356 |
+
""" """
|
| 357 |
+
pass
|
| 358 |
+
|
| 359 |
+
@property
|
| 360 |
+
def split(self):
|
| 361 |
+
""" """
|
| 362 |
+
pass
|
| 363 |
+
|
| 364 |
+
@split.setter
|
| 365 |
+
def split(self, value):
|
| 366 |
+
""" """
|
| 367 |
+
pass
|
| 368 |
+
|
| 369 |
+
class Replace(Decoder):
|
| 370 |
+
"""
|
| 371 |
+
Replace Decoder
|
| 372 |
+
|
| 373 |
+
This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.Replace`
|
| 374 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`.
|
| 375 |
+
"""
|
| 376 |
+
def __init__(self, pattern, content):
|
| 377 |
+
pass
|
| 378 |
+
|
| 379 |
+
def __getstate__(self):
|
| 380 |
+
""" """
|
| 381 |
+
pass
|
| 382 |
+
|
| 383 |
+
def __setstate__(self, state):
|
| 384 |
+
""" """
|
| 385 |
+
pass
|
| 386 |
+
|
| 387 |
+
@staticmethod
|
| 388 |
+
def custom(decoder):
|
| 389 |
+
""" """
|
| 390 |
+
pass
|
| 391 |
+
|
| 392 |
+
def decode(self, tokens):
|
| 393 |
+
"""
|
| 394 |
+
Decode the given list of tokens to a final string
|
| 395 |
+
|
| 396 |
+
Args:
|
| 397 |
+
tokens (:obj:`List[str]`):
|
| 398 |
+
The list of tokens to decode
|
| 399 |
+
|
| 400 |
+
Returns:
|
| 401 |
+
:obj:`str`: The decoded string
|
| 402 |
+
"""
|
| 403 |
+
pass
|
| 404 |
+
|
| 405 |
+
class Sequence(Decoder):
|
| 406 |
+
"""
|
| 407 |
+
Sequence Decoder
|
| 408 |
+
|
| 409 |
+
Args:
|
| 410 |
+
decoders (:obj:`List[Decoder]`)
|
| 411 |
+
The decoders that need to be chained
|
| 412 |
+
"""
|
| 413 |
+
def __init__(self, decoders):
|
| 414 |
+
pass
|
| 415 |
+
|
| 416 |
+
def __getnewargs__(self):
|
| 417 |
+
""" """
|
| 418 |
+
pass
|
| 419 |
+
|
| 420 |
+
def __getstate__(self):
|
| 421 |
+
""" """
|
| 422 |
+
pass
|
| 423 |
+
|
| 424 |
+
def __setstate__(self, state):
|
| 425 |
+
""" """
|
| 426 |
+
pass
|
| 427 |
+
|
| 428 |
+
@staticmethod
|
| 429 |
+
def custom(decoder):
|
| 430 |
+
""" """
|
| 431 |
+
pass
|
| 432 |
+
|
| 433 |
+
def decode(self, tokens):
|
| 434 |
+
"""
|
| 435 |
+
Decode the given list of tokens to a final string
|
| 436 |
+
|
| 437 |
+
Args:
|
| 438 |
+
tokens (:obj:`List[str]`):
|
| 439 |
+
The list of tokens to decode
|
| 440 |
+
|
| 441 |
+
Returns:
|
| 442 |
+
:obj:`str`: The decoded string
|
| 443 |
+
"""
|
| 444 |
+
pass
|
| 445 |
+
|
| 446 |
+
class Strip(Decoder):
|
| 447 |
+
"""
|
| 448 |
+
Strip normalizer
|
| 449 |
+
Strips n left characters of each token, or n right characters of each token
|
| 450 |
+
"""
|
| 451 |
+
def __init__(self, content=" ", left=0, right=0):
|
| 452 |
+
pass
|
| 453 |
+
|
| 454 |
+
def __getstate__(self):
|
| 455 |
+
""" """
|
| 456 |
+
pass
|
| 457 |
+
|
| 458 |
+
def __setstate__(self, state):
|
| 459 |
+
""" """
|
| 460 |
+
pass
|
| 461 |
+
|
| 462 |
+
@property
|
| 463 |
+
def content(self):
|
| 464 |
+
""" """
|
| 465 |
+
pass
|
| 466 |
+
|
| 467 |
+
@content.setter
|
| 468 |
+
def content(self, value):
|
| 469 |
+
""" """
|
| 470 |
+
pass
|
| 471 |
+
|
| 472 |
+
@staticmethod
|
| 473 |
+
def custom(decoder):
|
| 474 |
+
""" """
|
| 475 |
+
pass
|
| 476 |
+
|
| 477 |
+
def decode(self, tokens):
|
| 478 |
+
"""
|
| 479 |
+
Decode the given list of tokens to a final string
|
| 480 |
+
|
| 481 |
+
Args:
|
| 482 |
+
tokens (:obj:`List[str]`):
|
| 483 |
+
The list of tokens to decode
|
| 484 |
+
|
| 485 |
+
Returns:
|
| 486 |
+
:obj:`str`: The decoded string
|
| 487 |
+
"""
|
| 488 |
+
pass
|
| 489 |
+
|
| 490 |
+
@property
|
| 491 |
+
def start(self):
|
| 492 |
+
""" """
|
| 493 |
+
pass
|
| 494 |
+
|
| 495 |
+
@start.setter
|
| 496 |
+
def start(self, value):
|
| 497 |
+
""" """
|
| 498 |
+
pass
|
| 499 |
+
|
| 500 |
+
@property
|
| 501 |
+
def stop(self):
|
| 502 |
+
""" """
|
| 503 |
+
pass
|
| 504 |
+
|
| 505 |
+
@stop.setter
|
| 506 |
+
def stop(self, value):
|
| 507 |
+
""" """
|
| 508 |
+
pass
|
| 509 |
+
|
| 510 |
+
class WordPiece(Decoder):
|
| 511 |
+
"""
|
| 512 |
+
WordPiece Decoder
|
| 513 |
+
|
| 514 |
+
Args:
|
| 515 |
+
prefix (:obj:`str`, `optional`, defaults to :obj:`##`):
|
| 516 |
+
The prefix to use for subwords that are not a beginning-of-word
|
| 517 |
+
|
| 518 |
+
cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 519 |
+
Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation,
|
| 520 |
+
and some abbreviated english forms.
|
| 521 |
+
"""
|
| 522 |
+
def __init__(self, prefix="##", cleanup=True):
|
| 523 |
+
pass
|
| 524 |
+
|
| 525 |
+
def __getstate__(self):
|
| 526 |
+
""" """
|
| 527 |
+
pass
|
| 528 |
+
|
| 529 |
+
def __setstate__(self, state):
|
| 530 |
+
""" """
|
| 531 |
+
pass
|
| 532 |
+
|
| 533 |
+
@property
|
| 534 |
+
def cleanup(self):
|
| 535 |
+
""" """
|
| 536 |
+
pass
|
| 537 |
+
|
| 538 |
+
@cleanup.setter
|
| 539 |
+
def cleanup(self, value):
|
| 540 |
+
""" """
|
| 541 |
+
pass
|
| 542 |
+
|
| 543 |
+
@staticmethod
|
| 544 |
+
def custom(decoder):
|
| 545 |
+
""" """
|
| 546 |
+
pass
|
| 547 |
+
|
| 548 |
+
def decode(self, tokens):
|
| 549 |
+
"""
|
| 550 |
+
Decode the given list of tokens to a final string
|
| 551 |
+
|
| 552 |
+
Args:
|
| 553 |
+
tokens (:obj:`List[str]`):
|
| 554 |
+
The list of tokens to decode
|
| 555 |
+
|
| 556 |
+
Returns:
|
| 557 |
+
:obj:`str`: The decoded string
|
| 558 |
+
"""
|
| 559 |
+
pass
|
| 560 |
+
|
| 561 |
+
@property
|
| 562 |
+
def prefix(self):
|
| 563 |
+
""" """
|
| 564 |
+
pass
|
| 565 |
+
|
| 566 |
+
@prefix.setter
|
| 567 |
+
def prefix(self, value):
|
| 568 |
+
""" """
|
| 569 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .base_tokenizer import BaseTokenizer
|
| 2 |
+
from .bert_wordpiece import BertWordPieceTokenizer
|
| 3 |
+
from .byte_level_bpe import ByteLevelBPETokenizer
|
| 4 |
+
from .char_level_bpe import CharBPETokenizer
|
| 5 |
+
from .sentencepiece_bpe import SentencePieceBPETokenizer
|
| 6 |
+
from .sentencepiece_unigram import SentencePieceUnigramTokenizer
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/base_tokenizer.py
ADDED
|
@@ -0,0 +1,459 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, List, Optional, Tuple, Union
|
| 2 |
+
|
| 3 |
+
from tokenizers import AddedToken, EncodeInput, Encoding, InputSequence, Tokenizer
|
| 4 |
+
from tokenizers.decoders import Decoder
|
| 5 |
+
from tokenizers.models import Model
|
| 6 |
+
from tokenizers.normalizers import Normalizer
|
| 7 |
+
from tokenizers.pre_tokenizers import PreTokenizer
|
| 8 |
+
from tokenizers.processors import PostProcessor
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
Offsets = Tuple[int, int]
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class BaseTokenizer:
|
| 15 |
+
def __init__(self, tokenizer: Tokenizer, parameters=None):
|
| 16 |
+
self._tokenizer = tokenizer
|
| 17 |
+
self._parameters = parameters if parameters is not None else {}
|
| 18 |
+
|
| 19 |
+
def __repr__(self):
|
| 20 |
+
return "Tokenizer(vocabulary_size={}, {})".format(
|
| 21 |
+
self._tokenizer.get_vocab_size(),
|
| 22 |
+
", ".join(k + "=" + str(v) for k, v in self._parameters.items()),
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
def num_special_tokens_to_add(self, is_pair: bool) -> int:
|
| 26 |
+
"""
|
| 27 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 28 |
+
:param is_pair: Boolean indicating if the input would be a single sentence or a pair
|
| 29 |
+
:return:
|
| 30 |
+
"""
|
| 31 |
+
return self._tokenizer.num_special_tokens_to_add(is_pair)
|
| 32 |
+
|
| 33 |
+
def get_vocab(self, with_added_tokens: bool = True) -> Dict[str, int]:
|
| 34 |
+
"""Returns the vocabulary
|
| 35 |
+
|
| 36 |
+
Args:
|
| 37 |
+
with_added_tokens: boolean:
|
| 38 |
+
Whether to include the added tokens in the vocabulary
|
| 39 |
+
|
| 40 |
+
Returns:
|
| 41 |
+
The vocabulary
|
| 42 |
+
"""
|
| 43 |
+
return self._tokenizer.get_vocab(with_added_tokens=with_added_tokens)
|
| 44 |
+
|
| 45 |
+
def get_added_tokens_decoder(self) -> Dict[int, AddedToken]:
|
| 46 |
+
"""Returns the added reverse vocabulary
|
| 47 |
+
|
| 48 |
+
Returns:
|
| 49 |
+
The added vocabulary mapping ints to AddedTokens
|
| 50 |
+
"""
|
| 51 |
+
return self._tokenizer.get_added_tokens_decoder()
|
| 52 |
+
|
| 53 |
+
def get_vocab_size(self, with_added_tokens: bool = True) -> int:
|
| 54 |
+
"""Return the size of vocabulary, with or without added tokens.
|
| 55 |
+
|
| 56 |
+
Args:
|
| 57 |
+
with_added_tokens: (`optional`) bool:
|
| 58 |
+
Whether to count in added special tokens or not
|
| 59 |
+
|
| 60 |
+
Returns:
|
| 61 |
+
Size of vocabulary
|
| 62 |
+
"""
|
| 63 |
+
return self._tokenizer.get_vocab_size(with_added_tokens=with_added_tokens)
|
| 64 |
+
|
| 65 |
+
def enable_padding(
|
| 66 |
+
self,
|
| 67 |
+
direction: Optional[str] = "right",
|
| 68 |
+
pad_to_multiple_of: Optional[int] = None,
|
| 69 |
+
pad_id: Optional[int] = 0,
|
| 70 |
+
pad_type_id: Optional[int] = 0,
|
| 71 |
+
pad_token: Optional[str] = "[PAD]",
|
| 72 |
+
length: Optional[int] = None,
|
| 73 |
+
):
|
| 74 |
+
"""Change the padding strategy
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
direction: (`optional`) str:
|
| 78 |
+
Can be one of: `right` or `left`
|
| 79 |
+
|
| 80 |
+
pad_to_multiple_of: (`optional`) unsigned int:
|
| 81 |
+
If specified, the padding length should always snap to the next multiple of
|
| 82 |
+
the given value. For example if we were going to pad with a length of 250 but
|
| 83 |
+
`pad_to_multiple_of=8` then we will pad to 256.
|
| 84 |
+
|
| 85 |
+
pad_id: (`optional`) unsigned int:
|
| 86 |
+
The indice to be used when padding
|
| 87 |
+
|
| 88 |
+
pad_type_id: (`optional`) unsigned int:
|
| 89 |
+
The type indice to be used when padding
|
| 90 |
+
|
| 91 |
+
pad_token: (`optional`) str:
|
| 92 |
+
The pad token to be used when padding
|
| 93 |
+
|
| 94 |
+
length: (`optional`) unsigned int:
|
| 95 |
+
If specified, the length at which to pad. If not specified
|
| 96 |
+
we pad using the size of the longest sequence in a batch
|
| 97 |
+
"""
|
| 98 |
+
return self._tokenizer.enable_padding(
|
| 99 |
+
direction=direction,
|
| 100 |
+
pad_to_multiple_of=pad_to_multiple_of,
|
| 101 |
+
pad_id=pad_id,
|
| 102 |
+
pad_type_id=pad_type_id,
|
| 103 |
+
pad_token=pad_token,
|
| 104 |
+
length=length,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def no_padding(self):
|
| 108 |
+
"""Disable padding"""
|
| 109 |
+
return self._tokenizer.no_padding()
|
| 110 |
+
|
| 111 |
+
@property
|
| 112 |
+
def padding(self) -> Optional[dict]:
|
| 113 |
+
"""Get the current padding parameters
|
| 114 |
+
|
| 115 |
+
Returns:
|
| 116 |
+
None if padding is disabled, a dict with the currently set parameters
|
| 117 |
+
if the padding is enabled.
|
| 118 |
+
"""
|
| 119 |
+
return self._tokenizer.padding
|
| 120 |
+
|
| 121 |
+
def enable_truncation(self, max_length: int, stride: Optional[int] = 0, strategy: Optional[str] = "longest_first"):
|
| 122 |
+
"""Change the truncation options
|
| 123 |
+
|
| 124 |
+
Args:
|
| 125 |
+
max_length: unsigned int:
|
| 126 |
+
The maximum length at which to truncate
|
| 127 |
+
|
| 128 |
+
stride: (`optional`) unsigned int:
|
| 129 |
+
The length of the previous first sequence to be included
|
| 130 |
+
in the overflowing sequence
|
| 131 |
+
|
| 132 |
+
strategy: (`optional`) str:
|
| 133 |
+
Can be one of `longest_first`, `only_first` or `only_second`
|
| 134 |
+
"""
|
| 135 |
+
return self._tokenizer.enable_truncation(max_length, stride=stride, strategy=strategy)
|
| 136 |
+
|
| 137 |
+
def no_truncation(self):
|
| 138 |
+
"""Disable truncation"""
|
| 139 |
+
return self._tokenizer.no_truncation()
|
| 140 |
+
|
| 141 |
+
@property
|
| 142 |
+
def truncation(self) -> Optional[dict]:
|
| 143 |
+
"""Get the current truncation parameters
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
None if truncation is disabled, a dict with the current truncation parameters if
|
| 147 |
+
truncation is enabled
|
| 148 |
+
"""
|
| 149 |
+
return self._tokenizer.truncation
|
| 150 |
+
|
| 151 |
+
def add_tokens(self, tokens: List[Union[str, AddedToken]]) -> int:
|
| 152 |
+
"""Add the given tokens to the vocabulary
|
| 153 |
+
|
| 154 |
+
Args:
|
| 155 |
+
tokens: List[Union[str, AddedToken]]:
|
| 156 |
+
A list of tokens to add to the vocabulary. Each token can either be
|
| 157 |
+
a string, or an instance of AddedToken
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
The number of tokens that were added to the vocabulary
|
| 161 |
+
"""
|
| 162 |
+
return self._tokenizer.add_tokens(tokens)
|
| 163 |
+
|
| 164 |
+
def add_special_tokens(self, special_tokens: List[Union[str, AddedToken]]) -> int:
|
| 165 |
+
"""Add the given special tokens to the vocabulary, and treat them as special tokens.
|
| 166 |
+
|
| 167 |
+
The special tokens will never be processed by the model, and will be
|
| 168 |
+
removed while decoding.
|
| 169 |
+
|
| 170 |
+
Args:
|
| 171 |
+
tokens: List[Union[str, AddedToken]]:
|
| 172 |
+
A list of special tokens to add to the vocabulary. Each token can either be
|
| 173 |
+
a string, or an instance of AddedToken
|
| 174 |
+
|
| 175 |
+
Returns:
|
| 176 |
+
The number of tokens that were added to the vocabulary
|
| 177 |
+
"""
|
| 178 |
+
return self._tokenizer.add_special_tokens(special_tokens)
|
| 179 |
+
|
| 180 |
+
def normalize(self, sequence: str) -> str:
|
| 181 |
+
"""Normalize the given sequence
|
| 182 |
+
|
| 183 |
+
Args:
|
| 184 |
+
sequence: str:
|
| 185 |
+
The sequence to normalize
|
| 186 |
+
|
| 187 |
+
Returns:
|
| 188 |
+
The normalized string
|
| 189 |
+
"""
|
| 190 |
+
return self._tokenizer.normalizer.normalize_str(sequence)
|
| 191 |
+
|
| 192 |
+
def encode(
|
| 193 |
+
self,
|
| 194 |
+
sequence: InputSequence,
|
| 195 |
+
pair: Optional[InputSequence] = None,
|
| 196 |
+
is_pretokenized: bool = False,
|
| 197 |
+
add_special_tokens: bool = True,
|
| 198 |
+
) -> Encoding:
|
| 199 |
+
"""Encode the given sequence and pair. This method can process raw text sequences as well
|
| 200 |
+
as already pre-tokenized sequences.
|
| 201 |
+
|
| 202 |
+
Args:
|
| 203 |
+
sequence: InputSequence:
|
| 204 |
+
The sequence we want to encode. This sequence can be either raw text or
|
| 205 |
+
pre-tokenized, according to the `is_pretokenized` argument:
|
| 206 |
+
|
| 207 |
+
- If `is_pretokenized=False`: `InputSequence` is expected to be `str`
|
| 208 |
+
- If `is_pretokenized=True`: `InputSequence` is expected to be
|
| 209 |
+
`Union[List[str], Tuple[str]]`
|
| 210 |
+
|
| 211 |
+
is_pretokenized: bool:
|
| 212 |
+
Whether the input is already pre-tokenized.
|
| 213 |
+
|
| 214 |
+
add_special_tokens: bool:
|
| 215 |
+
Whether to add the special tokens while encoding.
|
| 216 |
+
|
| 217 |
+
Returns:
|
| 218 |
+
An Encoding
|
| 219 |
+
"""
|
| 220 |
+
if sequence is None:
|
| 221 |
+
raise ValueError("encode: `sequence` can't be `None`")
|
| 222 |
+
|
| 223 |
+
return self._tokenizer.encode(sequence, pair, is_pretokenized, add_special_tokens)
|
| 224 |
+
|
| 225 |
+
def encode_batch(
|
| 226 |
+
self,
|
| 227 |
+
inputs: List[EncodeInput],
|
| 228 |
+
is_pretokenized: bool = False,
|
| 229 |
+
add_special_tokens: bool = True,
|
| 230 |
+
) -> List[Encoding]:
|
| 231 |
+
"""Encode the given inputs. This method accept both raw text sequences as well as already
|
| 232 |
+
pre-tokenized sequences.
|
| 233 |
+
|
| 234 |
+
Args:
|
| 235 |
+
inputs: List[EncodeInput]:
|
| 236 |
+
A list of single sequences or pair sequences to encode. Each `EncodeInput` is
|
| 237 |
+
expected to be of the following form:
|
| 238 |
+
`Union[InputSequence, Tuple[InputSequence, InputSequence]]`
|
| 239 |
+
|
| 240 |
+
Each `InputSequence` can either be raw text or pre-tokenized,
|
| 241 |
+
according to the `is_pretokenized` argument:
|
| 242 |
+
|
| 243 |
+
- If `is_pretokenized=False`: `InputSequence` is expected to be `str`
|
| 244 |
+
- If `is_pretokenized=True`: `InputSequence` is expected to be
|
| 245 |
+
`Union[List[str], Tuple[str]]`
|
| 246 |
+
|
| 247 |
+
is_pretokenized: bool:
|
| 248 |
+
Whether the input is already pre-tokenized.
|
| 249 |
+
|
| 250 |
+
add_special_tokens: bool:
|
| 251 |
+
Whether to add the special tokens while encoding.
|
| 252 |
+
|
| 253 |
+
Returns:
|
| 254 |
+
A list of Encoding
|
| 255 |
+
"""
|
| 256 |
+
|
| 257 |
+
if inputs is None:
|
| 258 |
+
raise ValueError("encode_batch: `inputs` can't be `None`")
|
| 259 |
+
|
| 260 |
+
return self._tokenizer.encode_batch(inputs, is_pretokenized, add_special_tokens)
|
| 261 |
+
|
| 262 |
+
async def async_encode_batch(
|
| 263 |
+
self,
|
| 264 |
+
inputs: List[EncodeInput],
|
| 265 |
+
is_pretokenized: bool = False,
|
| 266 |
+
add_special_tokens: bool = True,
|
| 267 |
+
) -> List[Encoding]:
|
| 268 |
+
"""Asynchronously encode a batch (tracks character offsets).
|
| 269 |
+
|
| 270 |
+
Args:
|
| 271 |
+
inputs: A list of single or pair sequences to encode.
|
| 272 |
+
is_pretokenized: Whether inputs are already pre-tokenized.
|
| 273 |
+
add_special_tokens: Whether to add special tokens.
|
| 274 |
+
|
| 275 |
+
Returns:
|
| 276 |
+
A list of Encoding.
|
| 277 |
+
"""
|
| 278 |
+
if inputs is None:
|
| 279 |
+
raise ValueError("async_encode_batch: `inputs` can't be `None`")
|
| 280 |
+
# Exposed by the Rust bindings via pyo3_async_runtimes::tokio::future_into_py
|
| 281 |
+
return await self._tokenizer.async_encode_batch(inputs, is_pretokenized, add_special_tokens)
|
| 282 |
+
|
| 283 |
+
async def async_encode_batch_fast(
|
| 284 |
+
self,
|
| 285 |
+
inputs: List[EncodeInput],
|
| 286 |
+
is_pretokenized: bool = False,
|
| 287 |
+
add_special_tokens: bool = True,
|
| 288 |
+
) -> List[Encoding]:
|
| 289 |
+
"""Asynchronously encode a batch (no character offsets, faster).
|
| 290 |
+
|
| 291 |
+
Args:
|
| 292 |
+
inputs: A list of single or pair sequences to encode.
|
| 293 |
+
is_pretokenized: Whether inputs are already pre-tokenized.
|
| 294 |
+
add_special_tokens: Whether to add special tokens.
|
| 295 |
+
|
| 296 |
+
Returns:
|
| 297 |
+
A list of Encoding.
|
| 298 |
+
"""
|
| 299 |
+
if inputs is None:
|
| 300 |
+
raise ValueError("async_encode_batch_fast: `inputs` can't be `None`")
|
| 301 |
+
return await self._tokenizer.async_encode_batch_fast(inputs, is_pretokenized, add_special_tokens)
|
| 302 |
+
|
| 303 |
+
def decode(self, ids: List[int], skip_special_tokens: Optional[bool] = True) -> str:
|
| 304 |
+
"""Decode the given list of ids to a string sequence
|
| 305 |
+
|
| 306 |
+
Args:
|
| 307 |
+
ids: List[unsigned int]:
|
| 308 |
+
A list of ids to be decoded
|
| 309 |
+
|
| 310 |
+
skip_special_tokens: (`optional`) boolean:
|
| 311 |
+
Whether to remove all the special tokens from the output string
|
| 312 |
+
|
| 313 |
+
Returns:
|
| 314 |
+
The decoded string
|
| 315 |
+
"""
|
| 316 |
+
if ids is None:
|
| 317 |
+
raise ValueError("None input is not valid. Should be a list of integers.")
|
| 318 |
+
|
| 319 |
+
return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
|
| 320 |
+
|
| 321 |
+
def decode_batch(self, sequences: List[List[int]], skip_special_tokens: Optional[bool] = True) -> str:
|
| 322 |
+
"""Decode the list of sequences to a list of string sequences
|
| 323 |
+
|
| 324 |
+
Args:
|
| 325 |
+
sequences: List[List[unsigned int]]:
|
| 326 |
+
A list of sequence of ids to be decoded
|
| 327 |
+
|
| 328 |
+
skip_special_tokens: (`optional`) boolean:
|
| 329 |
+
Whether to remove all the special tokens from the output strings
|
| 330 |
+
|
| 331 |
+
Returns:
|
| 332 |
+
A list of decoded strings
|
| 333 |
+
"""
|
| 334 |
+
if sequences is None:
|
| 335 |
+
raise ValueError("None input is not valid. Should be list of list of integers.")
|
| 336 |
+
|
| 337 |
+
return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens)
|
| 338 |
+
|
| 339 |
+
def token_to_id(self, token: str) -> Optional[int]:
|
| 340 |
+
"""Convert the given token to its corresponding id
|
| 341 |
+
|
| 342 |
+
Args:
|
| 343 |
+
token: str:
|
| 344 |
+
The token to convert
|
| 345 |
+
|
| 346 |
+
Returns:
|
| 347 |
+
The corresponding id if it exists, None otherwise
|
| 348 |
+
"""
|
| 349 |
+
return self._tokenizer.token_to_id(token)
|
| 350 |
+
|
| 351 |
+
def id_to_token(self, id: int) -> Optional[str]:
|
| 352 |
+
"""Convert the given token id to its corresponding string
|
| 353 |
+
|
| 354 |
+
Args:
|
| 355 |
+
token: id:
|
| 356 |
+
The token id to convert
|
| 357 |
+
|
| 358 |
+
Returns:
|
| 359 |
+
The corresponding string if it exists, None otherwise
|
| 360 |
+
"""
|
| 361 |
+
return self._tokenizer.id_to_token(id)
|
| 362 |
+
|
| 363 |
+
def save_model(self, directory: str, prefix: Optional[str] = None):
|
| 364 |
+
"""Save the current model to the given directory
|
| 365 |
+
|
| 366 |
+
Args:
|
| 367 |
+
directory: str:
|
| 368 |
+
A path to the destination directory
|
| 369 |
+
|
| 370 |
+
prefix: (Optional) str:
|
| 371 |
+
An optional prefix, used to prefix each file name
|
| 372 |
+
"""
|
| 373 |
+
return self._tokenizer.model.save(directory, prefix=prefix)
|
| 374 |
+
|
| 375 |
+
def save(self, path: str, pretty: bool = True):
|
| 376 |
+
"""Save the current Tokenizer at the given path
|
| 377 |
+
|
| 378 |
+
Args:
|
| 379 |
+
path: str:
|
| 380 |
+
A path to the destination Tokenizer file
|
| 381 |
+
"""
|
| 382 |
+
return self._tokenizer.save(path, pretty)
|
| 383 |
+
|
| 384 |
+
def to_str(self, pretty: bool = False):
|
| 385 |
+
"""Get a serialized JSON version of the Tokenizer as a str
|
| 386 |
+
|
| 387 |
+
Args:
|
| 388 |
+
pretty: bool:
|
| 389 |
+
Whether the JSON string should be prettified
|
| 390 |
+
|
| 391 |
+
Returns:
|
| 392 |
+
str
|
| 393 |
+
"""
|
| 394 |
+
return self._tokenizer.to_str(pretty)
|
| 395 |
+
|
| 396 |
+
def post_process(
|
| 397 |
+
self, encoding: Encoding, pair: Optional[Encoding] = None, add_special_tokens: bool = True
|
| 398 |
+
) -> Encoding:
|
| 399 |
+
"""Apply all the post-processing steps to the given encodings.
|
| 400 |
+
|
| 401 |
+
The various steps are:
|
| 402 |
+
1. Truncate according to global params (provided to `enable_truncation`)
|
| 403 |
+
2. Apply the PostProcessor
|
| 404 |
+
3. Pad according to global params. (provided to `enable_padding`)
|
| 405 |
+
|
| 406 |
+
Args:
|
| 407 |
+
encoding: Encoding:
|
| 408 |
+
The main Encoding to post process
|
| 409 |
+
|
| 410 |
+
pair: Optional[Encoding]:
|
| 411 |
+
An optional pair Encoding
|
| 412 |
+
|
| 413 |
+
add_special_tokens: bool:
|
| 414 |
+
Whether to add special tokens
|
| 415 |
+
|
| 416 |
+
Returns:
|
| 417 |
+
The resulting Encoding
|
| 418 |
+
"""
|
| 419 |
+
return self._tokenizer.post_process(encoding, pair, add_special_tokens)
|
| 420 |
+
|
| 421 |
+
@property
|
| 422 |
+
def model(self) -> Model:
|
| 423 |
+
return self._tokenizer.model
|
| 424 |
+
|
| 425 |
+
@model.setter
|
| 426 |
+
def model(self, model: Model):
|
| 427 |
+
self._tokenizer.model = model
|
| 428 |
+
|
| 429 |
+
@property
|
| 430 |
+
def normalizer(self) -> Normalizer:
|
| 431 |
+
return self._tokenizer.normalizer
|
| 432 |
+
|
| 433 |
+
@normalizer.setter
|
| 434 |
+
def normalizer(self, normalizer: Normalizer):
|
| 435 |
+
self._tokenizer.normalizer = normalizer
|
| 436 |
+
|
| 437 |
+
@property
|
| 438 |
+
def pre_tokenizer(self) -> PreTokenizer:
|
| 439 |
+
return self._tokenizer.pre_tokenizer
|
| 440 |
+
|
| 441 |
+
@pre_tokenizer.setter
|
| 442 |
+
def pre_tokenizer(self, pre_tokenizer: PreTokenizer):
|
| 443 |
+
self._tokenizer.pre_tokenizer = pre_tokenizer
|
| 444 |
+
|
| 445 |
+
@property
|
| 446 |
+
def post_processor(self) -> PostProcessor:
|
| 447 |
+
return self._tokenizer.post_processor
|
| 448 |
+
|
| 449 |
+
@post_processor.setter
|
| 450 |
+
def post_processor(self, post_processor: PostProcessor):
|
| 451 |
+
self._tokenizer.post_processor = post_processor
|
| 452 |
+
|
| 453 |
+
@property
|
| 454 |
+
def decoder(self) -> Decoder:
|
| 455 |
+
return self._tokenizer.decoder
|
| 456 |
+
|
| 457 |
+
@decoder.setter
|
| 458 |
+
def decoder(self, decoder: Decoder):
|
| 459 |
+
self._tokenizer.decoder = decoder
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/bert_wordpiece.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Iterator, List, Optional, Union
|
| 2 |
+
|
| 3 |
+
from tokenizers import AddedToken, Tokenizer, decoders, trainers
|
| 4 |
+
from tokenizers.models import WordPiece
|
| 5 |
+
from tokenizers.normalizers import BertNormalizer
|
| 6 |
+
from tokenizers.pre_tokenizers import BertPreTokenizer
|
| 7 |
+
from tokenizers.processors import BertProcessing
|
| 8 |
+
|
| 9 |
+
from .base_tokenizer import BaseTokenizer
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class BertWordPieceTokenizer(BaseTokenizer):
|
| 13 |
+
"""Bert WordPiece Tokenizer"""
|
| 14 |
+
|
| 15 |
+
def __init__(
|
| 16 |
+
self,
|
| 17 |
+
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
| 18 |
+
unk_token: Union[str, AddedToken] = "[UNK]",
|
| 19 |
+
sep_token: Union[str, AddedToken] = "[SEP]",
|
| 20 |
+
cls_token: Union[str, AddedToken] = "[CLS]",
|
| 21 |
+
pad_token: Union[str, AddedToken] = "[PAD]",
|
| 22 |
+
mask_token: Union[str, AddedToken] = "[MASK]",
|
| 23 |
+
clean_text: bool = True,
|
| 24 |
+
handle_chinese_chars: bool = True,
|
| 25 |
+
strip_accents: Optional[bool] = None,
|
| 26 |
+
lowercase: bool = True,
|
| 27 |
+
wordpieces_prefix: str = "##",
|
| 28 |
+
):
|
| 29 |
+
if vocab is not None:
|
| 30 |
+
tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(unk_token)))
|
| 31 |
+
else:
|
| 32 |
+
tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token)))
|
| 33 |
+
|
| 34 |
+
# Let the tokenizer know about special tokens if they are part of the vocab
|
| 35 |
+
if tokenizer.token_to_id(str(unk_token)) is not None:
|
| 36 |
+
tokenizer.add_special_tokens([str(unk_token)])
|
| 37 |
+
if tokenizer.token_to_id(str(sep_token)) is not None:
|
| 38 |
+
tokenizer.add_special_tokens([str(sep_token)])
|
| 39 |
+
if tokenizer.token_to_id(str(cls_token)) is not None:
|
| 40 |
+
tokenizer.add_special_tokens([str(cls_token)])
|
| 41 |
+
if tokenizer.token_to_id(str(pad_token)) is not None:
|
| 42 |
+
tokenizer.add_special_tokens([str(pad_token)])
|
| 43 |
+
if tokenizer.token_to_id(str(mask_token)) is not None:
|
| 44 |
+
tokenizer.add_special_tokens([str(mask_token)])
|
| 45 |
+
|
| 46 |
+
tokenizer.normalizer = BertNormalizer(
|
| 47 |
+
clean_text=clean_text,
|
| 48 |
+
handle_chinese_chars=handle_chinese_chars,
|
| 49 |
+
strip_accents=strip_accents,
|
| 50 |
+
lowercase=lowercase,
|
| 51 |
+
)
|
| 52 |
+
tokenizer.pre_tokenizer = BertPreTokenizer()
|
| 53 |
+
|
| 54 |
+
if vocab is not None:
|
| 55 |
+
sep_token_id = tokenizer.token_to_id(str(sep_token))
|
| 56 |
+
if sep_token_id is None:
|
| 57 |
+
raise TypeError("sep_token not found in the vocabulary")
|
| 58 |
+
cls_token_id = tokenizer.token_to_id(str(cls_token))
|
| 59 |
+
if cls_token_id is None:
|
| 60 |
+
raise TypeError("cls_token not found in the vocabulary")
|
| 61 |
+
|
| 62 |
+
tokenizer.post_processor = BertProcessing((str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
|
| 63 |
+
tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
|
| 64 |
+
|
| 65 |
+
parameters = {
|
| 66 |
+
"model": "BertWordPiece",
|
| 67 |
+
"unk_token": unk_token,
|
| 68 |
+
"sep_token": sep_token,
|
| 69 |
+
"cls_token": cls_token,
|
| 70 |
+
"pad_token": pad_token,
|
| 71 |
+
"mask_token": mask_token,
|
| 72 |
+
"clean_text": clean_text,
|
| 73 |
+
"handle_chinese_chars": handle_chinese_chars,
|
| 74 |
+
"strip_accents": strip_accents,
|
| 75 |
+
"lowercase": lowercase,
|
| 76 |
+
"wordpieces_prefix": wordpieces_prefix,
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
super().__init__(tokenizer, parameters)
|
| 80 |
+
|
| 81 |
+
@staticmethod
|
| 82 |
+
def from_file(vocab: str, **kwargs):
|
| 83 |
+
vocab = WordPiece.read_file(vocab)
|
| 84 |
+
return BertWordPieceTokenizer(vocab, **kwargs)
|
| 85 |
+
|
| 86 |
+
def train(
|
| 87 |
+
self,
|
| 88 |
+
files: Union[str, List[str]],
|
| 89 |
+
vocab_size: int = 30000,
|
| 90 |
+
min_frequency: int = 2,
|
| 91 |
+
limit_alphabet: int = 1000,
|
| 92 |
+
initial_alphabet: List[str] = [],
|
| 93 |
+
special_tokens: List[Union[str, AddedToken]] = [
|
| 94 |
+
"[PAD]",
|
| 95 |
+
"[UNK]",
|
| 96 |
+
"[CLS]",
|
| 97 |
+
"[SEP]",
|
| 98 |
+
"[MASK]",
|
| 99 |
+
],
|
| 100 |
+
show_progress: bool = True,
|
| 101 |
+
wordpieces_prefix: str = "##",
|
| 102 |
+
):
|
| 103 |
+
"""Train the model using the given files"""
|
| 104 |
+
|
| 105 |
+
trainer = trainers.WordPieceTrainer(
|
| 106 |
+
vocab_size=vocab_size,
|
| 107 |
+
min_frequency=min_frequency,
|
| 108 |
+
limit_alphabet=limit_alphabet,
|
| 109 |
+
initial_alphabet=initial_alphabet,
|
| 110 |
+
special_tokens=special_tokens,
|
| 111 |
+
show_progress=show_progress,
|
| 112 |
+
continuing_subword_prefix=wordpieces_prefix,
|
| 113 |
+
)
|
| 114 |
+
if isinstance(files, str):
|
| 115 |
+
files = [files]
|
| 116 |
+
self._tokenizer.train(files, trainer=trainer)
|
| 117 |
+
|
| 118 |
+
def train_from_iterator(
|
| 119 |
+
self,
|
| 120 |
+
iterator: Union[Iterator[str], Iterator[Iterator[str]]],
|
| 121 |
+
vocab_size: int = 30000,
|
| 122 |
+
min_frequency: int = 2,
|
| 123 |
+
limit_alphabet: int = 1000,
|
| 124 |
+
initial_alphabet: List[str] = [],
|
| 125 |
+
special_tokens: List[Union[str, AddedToken]] = [
|
| 126 |
+
"[PAD]",
|
| 127 |
+
"[UNK]",
|
| 128 |
+
"[CLS]",
|
| 129 |
+
"[SEP]",
|
| 130 |
+
"[MASK]",
|
| 131 |
+
],
|
| 132 |
+
show_progress: bool = True,
|
| 133 |
+
wordpieces_prefix: str = "##",
|
| 134 |
+
length: Optional[int] = None,
|
| 135 |
+
):
|
| 136 |
+
"""Train the model using the given iterator"""
|
| 137 |
+
|
| 138 |
+
trainer = trainers.WordPieceTrainer(
|
| 139 |
+
vocab_size=vocab_size,
|
| 140 |
+
min_frequency=min_frequency,
|
| 141 |
+
limit_alphabet=limit_alphabet,
|
| 142 |
+
initial_alphabet=initial_alphabet,
|
| 143 |
+
special_tokens=special_tokens,
|
| 144 |
+
show_progress=show_progress,
|
| 145 |
+
continuing_subword_prefix=wordpieces_prefix,
|
| 146 |
+
)
|
| 147 |
+
self._tokenizer.train_from_iterator(
|
| 148 |
+
iterator,
|
| 149 |
+
trainer=trainer,
|
| 150 |
+
length=length,
|
| 151 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/byte_level_bpe.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Iterator, List, Optional, Tuple, Union
|
| 2 |
+
|
| 3 |
+
from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, processors, trainers
|
| 4 |
+
from tokenizers.models import BPE
|
| 5 |
+
from tokenizers.normalizers import Lowercase, Sequence, unicode_normalizer_from_str
|
| 6 |
+
|
| 7 |
+
from .base_tokenizer import BaseTokenizer
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class ByteLevelBPETokenizer(BaseTokenizer):
|
| 11 |
+
"""ByteLevelBPETokenizer
|
| 12 |
+
|
| 13 |
+
Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
| 19 |
+
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
|
| 20 |
+
add_prefix_space: bool = False,
|
| 21 |
+
lowercase: bool = False,
|
| 22 |
+
dropout: Optional[float] = None,
|
| 23 |
+
unicode_normalizer: Optional[str] = None,
|
| 24 |
+
continuing_subword_prefix: Optional[str] = None,
|
| 25 |
+
end_of_word_suffix: Optional[str] = None,
|
| 26 |
+
trim_offsets: bool = False,
|
| 27 |
+
):
|
| 28 |
+
if vocab is not None and merges is not None:
|
| 29 |
+
tokenizer = Tokenizer(
|
| 30 |
+
BPE(
|
| 31 |
+
vocab,
|
| 32 |
+
merges,
|
| 33 |
+
dropout=dropout,
|
| 34 |
+
continuing_subword_prefix=continuing_subword_prefix or "",
|
| 35 |
+
end_of_word_suffix=end_of_word_suffix or "",
|
| 36 |
+
)
|
| 37 |
+
)
|
| 38 |
+
else:
|
| 39 |
+
tokenizer = Tokenizer(BPE())
|
| 40 |
+
|
| 41 |
+
# Check for Unicode normalization first (before everything else)
|
| 42 |
+
normalizers = []
|
| 43 |
+
|
| 44 |
+
if unicode_normalizer:
|
| 45 |
+
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
|
| 46 |
+
|
| 47 |
+
if lowercase:
|
| 48 |
+
normalizers += [Lowercase()]
|
| 49 |
+
|
| 50 |
+
# Create the normalizer structure
|
| 51 |
+
if len(normalizers) > 0:
|
| 52 |
+
if len(normalizers) > 1:
|
| 53 |
+
tokenizer.normalizer = Sequence(normalizers)
|
| 54 |
+
else:
|
| 55 |
+
tokenizer.normalizer = normalizers[0]
|
| 56 |
+
|
| 57 |
+
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=add_prefix_space)
|
| 58 |
+
tokenizer.decoder = decoders.ByteLevel()
|
| 59 |
+
tokenizer.post_processor = processors.ByteLevel(trim_offsets=trim_offsets)
|
| 60 |
+
|
| 61 |
+
parameters = {
|
| 62 |
+
"model": "ByteLevelBPE",
|
| 63 |
+
"add_prefix_space": add_prefix_space,
|
| 64 |
+
"lowercase": lowercase,
|
| 65 |
+
"dropout": dropout,
|
| 66 |
+
"unicode_normalizer": unicode_normalizer,
|
| 67 |
+
"continuing_subword_prefix": continuing_subword_prefix,
|
| 68 |
+
"end_of_word_suffix": end_of_word_suffix,
|
| 69 |
+
"trim_offsets": trim_offsets,
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
super().__init__(tokenizer, parameters)
|
| 73 |
+
|
| 74 |
+
@staticmethod
|
| 75 |
+
def from_file(vocab_filename: str, merges_filename: str, **kwargs):
|
| 76 |
+
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
|
| 77 |
+
return ByteLevelBPETokenizer(vocab, merges, **kwargs)
|
| 78 |
+
|
| 79 |
+
def train(
|
| 80 |
+
self,
|
| 81 |
+
files: Union[str, List[str]],
|
| 82 |
+
vocab_size: int = 30000,
|
| 83 |
+
min_frequency: int = 2,
|
| 84 |
+
show_progress: bool = True,
|
| 85 |
+
special_tokens: List[Union[str, AddedToken]] = [],
|
| 86 |
+
):
|
| 87 |
+
"""Train the model using the given files"""
|
| 88 |
+
|
| 89 |
+
trainer = trainers.BpeTrainer(
|
| 90 |
+
vocab_size=vocab_size,
|
| 91 |
+
min_frequency=min_frequency,
|
| 92 |
+
show_progress=show_progress,
|
| 93 |
+
special_tokens=special_tokens,
|
| 94 |
+
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
|
| 95 |
+
)
|
| 96 |
+
if isinstance(files, str):
|
| 97 |
+
files = [files]
|
| 98 |
+
self._tokenizer.train(files, trainer=trainer)
|
| 99 |
+
|
| 100 |
+
def train_from_iterator(
|
| 101 |
+
self,
|
| 102 |
+
iterator: Union[Iterator[str], Iterator[Iterator[str]]],
|
| 103 |
+
vocab_size: int = 30000,
|
| 104 |
+
min_frequency: int = 2,
|
| 105 |
+
show_progress: bool = True,
|
| 106 |
+
special_tokens: List[Union[str, AddedToken]] = [],
|
| 107 |
+
length: Optional[int] = None,
|
| 108 |
+
):
|
| 109 |
+
"""Train the model using the given iterator"""
|
| 110 |
+
|
| 111 |
+
trainer = trainers.BpeTrainer(
|
| 112 |
+
vocab_size=vocab_size,
|
| 113 |
+
min_frequency=min_frequency,
|
| 114 |
+
show_progress=show_progress,
|
| 115 |
+
special_tokens=special_tokens,
|
| 116 |
+
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
|
| 117 |
+
)
|
| 118 |
+
self._tokenizer.train_from_iterator(
|
| 119 |
+
iterator,
|
| 120 |
+
trainer=trainer,
|
| 121 |
+
length=length,
|
| 122 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/char_level_bpe.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Iterator, List, Optional, Tuple, Union
|
| 2 |
+
|
| 3 |
+
from .. import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers
|
| 4 |
+
from ..models import BPE
|
| 5 |
+
from ..normalizers import BertNormalizer, Lowercase, Sequence, unicode_normalizer_from_str
|
| 6 |
+
from .base_tokenizer import BaseTokenizer
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class CharBPETokenizer(BaseTokenizer):
|
| 10 |
+
"""Original BPE Tokenizer
|
| 11 |
+
|
| 12 |
+
Represents the BPE algorithm, as introduced by Rico Sennrich
|
| 13 |
+
(https://arxiv.org/abs/1508.07909)
|
| 14 |
+
|
| 15 |
+
The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
|
| 16 |
+
Sennrich subword-nmt implementation by the following options that you can deactivate:
|
| 17 |
+
- adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
|
| 18 |
+
* removing any control characters and replacing all whitespaces by the classic one.
|
| 19 |
+
* handle chinese chars by putting spaces around them.
|
| 20 |
+
* strip all accents.
|
| 21 |
+
- spitting on punctuation in addition to whitespaces (deactivate it with
|
| 22 |
+
`split_on_whitespace_only=True`)
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
| 28 |
+
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
|
| 29 |
+
unk_token: Union[str, AddedToken] = "<unk>",
|
| 30 |
+
suffix: str = "</w>",
|
| 31 |
+
dropout: Optional[float] = None,
|
| 32 |
+
lowercase: bool = False,
|
| 33 |
+
unicode_normalizer: Optional[str] = None,
|
| 34 |
+
bert_normalizer: bool = True,
|
| 35 |
+
split_on_whitespace_only: bool = False,
|
| 36 |
+
):
|
| 37 |
+
if vocab is not None and merges is not None:
|
| 38 |
+
tokenizer = Tokenizer(
|
| 39 |
+
BPE(
|
| 40 |
+
vocab,
|
| 41 |
+
merges,
|
| 42 |
+
dropout=dropout,
|
| 43 |
+
unk_token=str(unk_token),
|
| 44 |
+
end_of_word_suffix=suffix,
|
| 45 |
+
)
|
| 46 |
+
)
|
| 47 |
+
else:
|
| 48 |
+
tokenizer = Tokenizer(BPE(unk_token=str(unk_token), dropout=dropout, end_of_word_suffix=suffix))
|
| 49 |
+
|
| 50 |
+
if tokenizer.token_to_id(str(unk_token)) is not None:
|
| 51 |
+
tokenizer.add_special_tokens([str(unk_token)])
|
| 52 |
+
|
| 53 |
+
# Check for Unicode normalization first (before everything else)
|
| 54 |
+
normalizers = []
|
| 55 |
+
|
| 56 |
+
if unicode_normalizer:
|
| 57 |
+
normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
|
| 58 |
+
|
| 59 |
+
if bert_normalizer:
|
| 60 |
+
normalizers += [BertNormalizer(lowercase=False)]
|
| 61 |
+
|
| 62 |
+
if lowercase:
|
| 63 |
+
normalizers += [Lowercase()]
|
| 64 |
+
|
| 65 |
+
# Create the normalizer structure
|
| 66 |
+
if len(normalizers) > 0:
|
| 67 |
+
if len(normalizers) > 1:
|
| 68 |
+
tokenizer.normalizer = Sequence(normalizers)
|
| 69 |
+
else:
|
| 70 |
+
tokenizer.normalizer = normalizers[0]
|
| 71 |
+
|
| 72 |
+
if split_on_whitespace_only:
|
| 73 |
+
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
| 74 |
+
else:
|
| 75 |
+
tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
|
| 76 |
+
|
| 77 |
+
tokenizer.decoder = decoders.BPEDecoder(suffix=suffix)
|
| 78 |
+
|
| 79 |
+
parameters = {
|
| 80 |
+
"model": "BPE",
|
| 81 |
+
"unk_token": unk_token,
|
| 82 |
+
"suffix": suffix,
|
| 83 |
+
"dropout": dropout,
|
| 84 |
+
"lowercase": lowercase,
|
| 85 |
+
"unicode_normalizer": unicode_normalizer,
|
| 86 |
+
"bert_normalizer": bert_normalizer,
|
| 87 |
+
"split_on_whitespace_only": split_on_whitespace_only,
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
super().__init__(tokenizer, parameters)
|
| 91 |
+
|
| 92 |
+
@staticmethod
|
| 93 |
+
def from_file(vocab_filename: str, merges_filename: str, **kwargs):
|
| 94 |
+
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
|
| 95 |
+
return CharBPETokenizer(vocab, merges, **kwargs)
|
| 96 |
+
|
| 97 |
+
def train(
|
| 98 |
+
self,
|
| 99 |
+
files: Union[str, List[str]],
|
| 100 |
+
vocab_size: int = 30000,
|
| 101 |
+
min_frequency: int = 2,
|
| 102 |
+
special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
|
| 103 |
+
limit_alphabet: int = 1000,
|
| 104 |
+
initial_alphabet: List[str] = [],
|
| 105 |
+
suffix: Optional[str] = "</w>",
|
| 106 |
+
show_progress: bool = True,
|
| 107 |
+
):
|
| 108 |
+
"""Train the model using the given files"""
|
| 109 |
+
|
| 110 |
+
trainer = trainers.BpeTrainer(
|
| 111 |
+
vocab_size=vocab_size,
|
| 112 |
+
min_frequency=min_frequency,
|
| 113 |
+
special_tokens=special_tokens,
|
| 114 |
+
limit_alphabet=limit_alphabet,
|
| 115 |
+
initial_alphabet=initial_alphabet,
|
| 116 |
+
end_of_word_suffix=suffix,
|
| 117 |
+
show_progress=show_progress,
|
| 118 |
+
)
|
| 119 |
+
if isinstance(files, str):
|
| 120 |
+
files = [files]
|
| 121 |
+
self._tokenizer.train(files, trainer=trainer)
|
| 122 |
+
|
| 123 |
+
def train_from_iterator(
|
| 124 |
+
self,
|
| 125 |
+
iterator: Union[Iterator[str], Iterator[Iterator[str]]],
|
| 126 |
+
vocab_size: int = 30000,
|
| 127 |
+
min_frequency: int = 2,
|
| 128 |
+
special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
|
| 129 |
+
limit_alphabet: int = 1000,
|
| 130 |
+
initial_alphabet: List[str] = [],
|
| 131 |
+
suffix: Optional[str] = "</w>",
|
| 132 |
+
show_progress: bool = True,
|
| 133 |
+
length: Optional[int] = None,
|
| 134 |
+
):
|
| 135 |
+
"""Train the model using the given iterator"""
|
| 136 |
+
|
| 137 |
+
trainer = trainers.BpeTrainer(
|
| 138 |
+
vocab_size=vocab_size,
|
| 139 |
+
min_frequency=min_frequency,
|
| 140 |
+
special_tokens=special_tokens,
|
| 141 |
+
limit_alphabet=limit_alphabet,
|
| 142 |
+
initial_alphabet=initial_alphabet,
|
| 143 |
+
end_of_word_suffix=suffix,
|
| 144 |
+
show_progress=show_progress,
|
| 145 |
+
)
|
| 146 |
+
self._tokenizer.train_from_iterator(
|
| 147 |
+
iterator,
|
| 148 |
+
trainer=trainer,
|
| 149 |
+
length=length,
|
| 150 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_bpe.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Iterator, List, Optional, Tuple, Union
|
| 2 |
+
|
| 3 |
+
from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers
|
| 4 |
+
from tokenizers.models import BPE
|
| 5 |
+
from tokenizers.normalizers import NFKC
|
| 6 |
+
|
| 7 |
+
from .base_tokenizer import BaseTokenizer
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class SentencePieceBPETokenizer(BaseTokenizer):
|
| 11 |
+
"""SentencePiece BPE Tokenizer
|
| 12 |
+
|
| 13 |
+
Represents the BPE algorithm, with the pretokenization used by SentencePiece
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
| 19 |
+
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
|
| 20 |
+
unk_token: Union[str, AddedToken] = "<unk>",
|
| 21 |
+
replacement: str = "▁",
|
| 22 |
+
add_prefix_space: bool = True,
|
| 23 |
+
dropout: Optional[float] = None,
|
| 24 |
+
fuse_unk: Optional[bool] = False,
|
| 25 |
+
):
|
| 26 |
+
if vocab is not None and merges is not None:
|
| 27 |
+
tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk))
|
| 28 |
+
else:
|
| 29 |
+
tokenizer = Tokenizer(BPE(dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk))
|
| 30 |
+
|
| 31 |
+
if tokenizer.token_to_id(str(unk_token)) is not None:
|
| 32 |
+
tokenizer.add_special_tokens([str(unk_token)])
|
| 33 |
+
|
| 34 |
+
tokenizer.normalizer = NFKC()
|
| 35 |
+
prepend_scheme = "always" if add_prefix_space else "never"
|
| 36 |
+
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
|
| 37 |
+
tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
|
| 38 |
+
|
| 39 |
+
parameters = {
|
| 40 |
+
"model": "SentencePieceBPE",
|
| 41 |
+
"unk_token": unk_token,
|
| 42 |
+
"replacement": replacement,
|
| 43 |
+
"add_prefix_space": add_prefix_space,
|
| 44 |
+
"dropout": dropout,
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
super().__init__(tokenizer, parameters)
|
| 48 |
+
|
| 49 |
+
@staticmethod
|
| 50 |
+
def from_file(vocab_filename: str, merges_filename: str, **kwargs):
|
| 51 |
+
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
|
| 52 |
+
return SentencePieceBPETokenizer(vocab, merges, **kwargs)
|
| 53 |
+
|
| 54 |
+
def train(
|
| 55 |
+
self,
|
| 56 |
+
files: Union[str, List[str]],
|
| 57 |
+
vocab_size: int = 30000,
|
| 58 |
+
min_frequency: int = 2,
|
| 59 |
+
special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
|
| 60 |
+
limit_alphabet: int = 1000,
|
| 61 |
+
initial_alphabet: List[str] = [],
|
| 62 |
+
show_progress: bool = True,
|
| 63 |
+
):
|
| 64 |
+
"""Train the model using the given files"""
|
| 65 |
+
|
| 66 |
+
trainer = trainers.BpeTrainer(
|
| 67 |
+
vocab_size=vocab_size,
|
| 68 |
+
min_frequency=min_frequency,
|
| 69 |
+
special_tokens=special_tokens,
|
| 70 |
+
limit_alphabet=limit_alphabet,
|
| 71 |
+
initial_alphabet=initial_alphabet,
|
| 72 |
+
show_progress=show_progress,
|
| 73 |
+
)
|
| 74 |
+
if isinstance(files, str):
|
| 75 |
+
files = [files]
|
| 76 |
+
self._tokenizer.train(files, trainer=trainer)
|
| 77 |
+
|
| 78 |
+
def train_from_iterator(
|
| 79 |
+
self,
|
| 80 |
+
iterator: Union[Iterator[str], Iterator[Iterator[str]]],
|
| 81 |
+
vocab_size: int = 30000,
|
| 82 |
+
min_frequency: int = 2,
|
| 83 |
+
special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
|
| 84 |
+
limit_alphabet: int = 1000,
|
| 85 |
+
initial_alphabet: List[str] = [],
|
| 86 |
+
show_progress: bool = True,
|
| 87 |
+
length: Optional[int] = None,
|
| 88 |
+
):
|
| 89 |
+
"""Train the model using the given iterator"""
|
| 90 |
+
|
| 91 |
+
trainer = trainers.BpeTrainer(
|
| 92 |
+
vocab_size=vocab_size,
|
| 93 |
+
min_frequency=min_frequency,
|
| 94 |
+
special_tokens=special_tokens,
|
| 95 |
+
limit_alphabet=limit_alphabet,
|
| 96 |
+
initial_alphabet=initial_alphabet,
|
| 97 |
+
show_progress=show_progress,
|
| 98 |
+
)
|
| 99 |
+
self._tokenizer.train_from_iterator(
|
| 100 |
+
iterator,
|
| 101 |
+
trainer=trainer,
|
| 102 |
+
length=length,
|
| 103 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_unigram.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
from typing import Iterator, List, Optional, Union, Tuple
|
| 4 |
+
|
| 5 |
+
from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
|
| 6 |
+
from tokenizers.models import Unigram
|
| 7 |
+
|
| 8 |
+
from .base_tokenizer import BaseTokenizer
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class SentencePieceUnigramTokenizer(BaseTokenizer):
|
| 12 |
+
"""SentencePiece Unigram Tokenizer
|
| 13 |
+
|
| 14 |
+
Represents the Unigram algorithm, with the pretokenization used by SentencePiece
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
vocab: Optional[List[Tuple[str, float]]] = None,
|
| 20 |
+
replacement: str = "▁",
|
| 21 |
+
add_prefix_space: bool = True,
|
| 22 |
+
):
|
| 23 |
+
if vocab is not None:
|
| 24 |
+
# Let Unigram(..) fail if only one of them is None
|
| 25 |
+
tokenizer = Tokenizer(Unigram(vocab))
|
| 26 |
+
else:
|
| 27 |
+
tokenizer = Tokenizer(Unigram())
|
| 28 |
+
|
| 29 |
+
tokenizer.normalizer = normalizers.Sequence(
|
| 30 |
+
[normalizers.Nmt(), normalizers.NFKC(), normalizers.Replace(Regex(" {2,}"), " ")]
|
| 31 |
+
)
|
| 32 |
+
prepend_scheme = "always" if add_prefix_space else "never"
|
| 33 |
+
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
|
| 34 |
+
tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
|
| 35 |
+
|
| 36 |
+
parameters = {
|
| 37 |
+
"model": "SentencePieceUnigram",
|
| 38 |
+
"replacement": replacement,
|
| 39 |
+
"add_prefix_space": add_prefix_space,
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
super().__init__(tokenizer, parameters)
|
| 43 |
+
|
| 44 |
+
def train(
|
| 45 |
+
self,
|
| 46 |
+
files: Union[str, List[str]],
|
| 47 |
+
vocab_size: int = 8000,
|
| 48 |
+
show_progress: bool = True,
|
| 49 |
+
special_tokens: Optional[List[Union[str, AddedToken]]] = None,
|
| 50 |
+
initial_alphabet: Optional[List[str]] = None,
|
| 51 |
+
unk_token: Optional[str] = None,
|
| 52 |
+
):
|
| 53 |
+
"""
|
| 54 |
+
Train the model using the given files
|
| 55 |
+
|
| 56 |
+
Args:
|
| 57 |
+
files (:obj:`List[str]`):
|
| 58 |
+
A list of path to the files that we should use for training
|
| 59 |
+
vocab_size (:obj:`int`):
|
| 60 |
+
The size of the final vocabulary, including all tokens and alphabet.
|
| 61 |
+
show_progress (:obj:`bool`):
|
| 62 |
+
Whether to show progress bars while training.
|
| 63 |
+
special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
|
| 64 |
+
A list of special tokens the model should know of.
|
| 65 |
+
initial_alphabet (:obj:`List[str]`, `optional`):
|
| 66 |
+
A list of characters to include in the initial alphabet, even
|
| 67 |
+
if not seen in the training dataset.
|
| 68 |
+
If the strings contain more than one character, only the first one
|
| 69 |
+
is kept.
|
| 70 |
+
unk_token (:obj:`str`, `optional`):
|
| 71 |
+
The unknown token to be used by the model.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
if special_tokens is None:
|
| 75 |
+
special_tokens = []
|
| 76 |
+
|
| 77 |
+
if initial_alphabet is None:
|
| 78 |
+
initial_alphabet = []
|
| 79 |
+
|
| 80 |
+
trainer = trainers.UnigramTrainer(
|
| 81 |
+
vocab_size=vocab_size,
|
| 82 |
+
special_tokens=special_tokens,
|
| 83 |
+
show_progress=show_progress,
|
| 84 |
+
initial_alphabet=initial_alphabet,
|
| 85 |
+
unk_token=unk_token,
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
if isinstance(files, str):
|
| 89 |
+
files = [files]
|
| 90 |
+
self._tokenizer.train(files, trainer=trainer)
|
| 91 |
+
|
| 92 |
+
def train_from_iterator(
|
| 93 |
+
self,
|
| 94 |
+
iterator: Union[Iterator[str], Iterator[Iterator[str]]],
|
| 95 |
+
vocab_size: int = 8000,
|
| 96 |
+
show_progress: bool = True,
|
| 97 |
+
special_tokens: Optional[List[Union[str, AddedToken]]] = None,
|
| 98 |
+
initial_alphabet: Optional[List[str]] = None,
|
| 99 |
+
unk_token: Optional[str] = None,
|
| 100 |
+
length: Optional[int] = None,
|
| 101 |
+
):
|
| 102 |
+
"""
|
| 103 |
+
Train the model using the given iterator
|
| 104 |
+
|
| 105 |
+
Args:
|
| 106 |
+
iterator (:obj:`Union[Iterator[str], Iterator[Iterator[str]]]`):
|
| 107 |
+
Any iterator over strings or list of strings
|
| 108 |
+
vocab_size (:obj:`int`):
|
| 109 |
+
The size of the final vocabulary, including all tokens and alphabet.
|
| 110 |
+
show_progress (:obj:`bool`):
|
| 111 |
+
Whether to show progress bars while training.
|
| 112 |
+
special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
|
| 113 |
+
A list of special tokens the model should know of.
|
| 114 |
+
initial_alphabet (:obj:`List[str]`, `optional`):
|
| 115 |
+
A list of characters to include in the initial alphabet, even
|
| 116 |
+
if not seen in the training dataset.
|
| 117 |
+
If the strings contain more than one character, only the first one
|
| 118 |
+
is kept.
|
| 119 |
+
unk_token (:obj:`str`, `optional`):
|
| 120 |
+
The unknown token to be used by the model.
|
| 121 |
+
length (:obj:`int`, `optional`):
|
| 122 |
+
The total number of sequences in the iterator. This is used to
|
| 123 |
+
provide meaningful progress tracking
|
| 124 |
+
"""
|
| 125 |
+
|
| 126 |
+
if special_tokens is None:
|
| 127 |
+
special_tokens = []
|
| 128 |
+
|
| 129 |
+
if initial_alphabet is None:
|
| 130 |
+
initial_alphabet = []
|
| 131 |
+
|
| 132 |
+
trainer = trainers.UnigramTrainer(
|
| 133 |
+
vocab_size=vocab_size,
|
| 134 |
+
special_tokens=special_tokens,
|
| 135 |
+
show_progress=show_progress,
|
| 136 |
+
initial_alphabet=initial_alphabet,
|
| 137 |
+
unk_token=unk_token,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
self._tokenizer.train_from_iterator(
|
| 141 |
+
iterator,
|
| 142 |
+
trainer=trainer,
|
| 143 |
+
length=length,
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
@staticmethod
|
| 147 |
+
def from_spm(filename: str):
|
| 148 |
+
try:
|
| 149 |
+
import sys
|
| 150 |
+
|
| 151 |
+
sys.path.append(".")
|
| 152 |
+
|
| 153 |
+
import sentencepiece_model_pb2 as model # type: ignore[import]
|
| 154 |
+
except Exception:
|
| 155 |
+
raise Exception(
|
| 156 |
+
"You don't seem to have the required protobuf file, in order to use this function you need to run `pip install protobuf` and `wget https://raw.githubusercontent.com/google/sentencepiece/master/python/src/sentencepiece/sentencepiece_model_pb2.py` for us to be able to read the intrinsics of your spm_file. `pip install sentencepiece` is not required."
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
m = model.ModelProto()
|
| 160 |
+
m.ParseFromString(open(filename, "rb").read())
|
| 161 |
+
|
| 162 |
+
precompiled_charsmap = m.normalizer_spec.precompiled_charsmap
|
| 163 |
+
vocab = [(piece.piece, piece.score) for piece in m.pieces]
|
| 164 |
+
unk_id = m.trainer_spec.unk_id
|
| 165 |
+
model_type = m.trainer_spec.model_type
|
| 166 |
+
byte_fallback = m.trainer_spec.byte_fallback
|
| 167 |
+
if model_type != 1:
|
| 168 |
+
raise Exception(
|
| 169 |
+
"You're trying to run a `Unigram` model but you're file was trained with a different algorithm"
|
| 170 |
+
)
|
| 171 |
+
|
| 172 |
+
replacement = "▁"
|
| 173 |
+
add_prefix_space = True
|
| 174 |
+
|
| 175 |
+
tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback))
|
| 176 |
+
|
| 177 |
+
if precompiled_charsmap:
|
| 178 |
+
tokenizer.normalizer = normalizers.Sequence(
|
| 179 |
+
[
|
| 180 |
+
normalizers.Precompiled(precompiled_charsmap),
|
| 181 |
+
normalizers.Replace(Regex(" {2,}"), " "),
|
| 182 |
+
]
|
| 183 |
+
)
|
| 184 |
+
else:
|
| 185 |
+
tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")])
|
| 186 |
+
prepend_scheme = "always" if add_prefix_space else "never"
|
| 187 |
+
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
|
| 188 |
+
tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
|
| 189 |
+
|
| 190 |
+
parameters = {
|
| 191 |
+
"model": "SentencePieceUnigram",
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
obj = BaseTokenizer.__new__(SentencePieceUnigramTokenizer, tokenizer, parameters) # type: ignore[arg-type]
|
| 195 |
+
BaseTokenizer.__init__(obj, tokenizer, parameters)
|
| 196 |
+
return obj
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/models/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
from .. import models
|
| 3 |
+
|
| 4 |
+
Model = models.Model
|
| 5 |
+
BPE = models.BPE
|
| 6 |
+
Unigram = models.Unigram
|
| 7 |
+
WordLevel = models.WordLevel
|
| 8 |
+
WordPiece = models.WordPiece
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/models/__init__.pyi
ADDED
|
@@ -0,0 +1,744 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class Model:
|
| 3 |
+
"""
|
| 4 |
+
Base class for all models
|
| 5 |
+
|
| 6 |
+
The model represents the actual tokenization algorithm. This is the part that
|
| 7 |
+
will contain and manage the learned vocabulary.
|
| 8 |
+
|
| 9 |
+
This class cannot be constructed directly. Please use one of the concrete models.
|
| 10 |
+
"""
|
| 11 |
+
def __init__(self):
|
| 12 |
+
pass
|
| 13 |
+
|
| 14 |
+
def __getstate__(self):
|
| 15 |
+
""" """
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
def __setstate__(self, state):
|
| 19 |
+
""" """
|
| 20 |
+
pass
|
| 21 |
+
|
| 22 |
+
def get_trainer(self):
|
| 23 |
+
"""
|
| 24 |
+
Get the associated :class:`~tokenizers.trainers.Trainer`
|
| 25 |
+
|
| 26 |
+
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
| 27 |
+
:class:`~tokenizers.models.Model`.
|
| 28 |
+
|
| 29 |
+
Returns:
|
| 30 |
+
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
| 31 |
+
"""
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
def id_to_token(self, id):
|
| 35 |
+
"""
|
| 36 |
+
Get the token associated to an ID
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
id (:obj:`int`):
|
| 40 |
+
An ID to convert to a token
|
| 41 |
+
|
| 42 |
+
Returns:
|
| 43 |
+
:obj:`str`: The token associated to the ID
|
| 44 |
+
"""
|
| 45 |
+
pass
|
| 46 |
+
|
| 47 |
+
def save(self, folder, prefix):
|
| 48 |
+
"""
|
| 49 |
+
Save the current model
|
| 50 |
+
|
| 51 |
+
Save the current model in the given folder, using the given prefix for the various
|
| 52 |
+
files that will get created.
|
| 53 |
+
Any file with the same name that already exists in this folder will be overwritten.
|
| 54 |
+
|
| 55 |
+
Args:
|
| 56 |
+
folder (:obj:`str`):
|
| 57 |
+
The path to the target folder in which to save the various files
|
| 58 |
+
|
| 59 |
+
prefix (:obj:`str`, `optional`):
|
| 60 |
+
An optional prefix, used to prefix each file name
|
| 61 |
+
|
| 62 |
+
Returns:
|
| 63 |
+
:obj:`List[str]`: The list of saved files
|
| 64 |
+
"""
|
| 65 |
+
pass
|
| 66 |
+
|
| 67 |
+
def token_to_id(self, tokens):
|
| 68 |
+
"""
|
| 69 |
+
Get the ID associated to a token
|
| 70 |
+
|
| 71 |
+
Args:
|
| 72 |
+
token (:obj:`str`):
|
| 73 |
+
A token to convert to an ID
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
:obj:`int`: The ID associated to the token
|
| 77 |
+
"""
|
| 78 |
+
pass
|
| 79 |
+
|
| 80 |
+
def tokenize(self, sequence):
|
| 81 |
+
"""
|
| 82 |
+
Tokenize a sequence
|
| 83 |
+
|
| 84 |
+
Args:
|
| 85 |
+
sequence (:obj:`str`):
|
| 86 |
+
A sequence to tokenize
|
| 87 |
+
|
| 88 |
+
Returns:
|
| 89 |
+
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
| 90 |
+
"""
|
| 91 |
+
pass
|
| 92 |
+
|
| 93 |
+
class BPE(Model):
|
| 94 |
+
"""
|
| 95 |
+
An implementation of the BPE (Byte-Pair Encoding) algorithm
|
| 96 |
+
|
| 97 |
+
Args:
|
| 98 |
+
vocab (:obj:`Dict[str, int]`, `optional`):
|
| 99 |
+
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
| 100 |
+
|
| 101 |
+
merges (:obj:`List[Tuple[str, str]]`, `optional`):
|
| 102 |
+
A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]`
|
| 103 |
+
|
| 104 |
+
cache_capacity (:obj:`int`, `optional`):
|
| 105 |
+
The number of words that the BPE cache can contain. The cache allows
|
| 106 |
+
to speed-up the process by keeping the result of the merge operations
|
| 107 |
+
for a number of words.
|
| 108 |
+
|
| 109 |
+
dropout (:obj:`float`, `optional`):
|
| 110 |
+
A float between 0 and 1 that represents the BPE dropout to use.
|
| 111 |
+
|
| 112 |
+
unk_token (:obj:`str`, `optional`):
|
| 113 |
+
The unknown token to be used by the model.
|
| 114 |
+
|
| 115 |
+
continuing_subword_prefix (:obj:`str`, `optional`):
|
| 116 |
+
The prefix to attach to subword units that don't represent a beginning of word.
|
| 117 |
+
|
| 118 |
+
end_of_word_suffix (:obj:`str`, `optional`):
|
| 119 |
+
The suffix to attach to subword units that represent an end of word.
|
| 120 |
+
|
| 121 |
+
fuse_unk (:obj:`bool`, `optional`):
|
| 122 |
+
Whether to fuse any subsequent unknown tokens into a single one
|
| 123 |
+
|
| 124 |
+
byte_fallback (:obj:`bool`, `optional`):
|
| 125 |
+
Whether to use spm byte-fallback trick (defaults to False)
|
| 126 |
+
|
| 127 |
+
ignore_merges (:obj:`bool`, `optional`):
|
| 128 |
+
Whether or not to match tokens with the vocab before using merges.
|
| 129 |
+
"""
|
| 130 |
+
def __init__(
|
| 131 |
+
self,
|
| 132 |
+
vocab=None,
|
| 133 |
+
merges=None,
|
| 134 |
+
cache_capacity=None,
|
| 135 |
+
dropout=None,
|
| 136 |
+
unk_token=None,
|
| 137 |
+
continuing_subword_prefix=None,
|
| 138 |
+
end_of_word_suffix=None,
|
| 139 |
+
fuse_unk=None,
|
| 140 |
+
byte_fallback=False,
|
| 141 |
+
ignore_merges=False,
|
| 142 |
+
):
|
| 143 |
+
pass
|
| 144 |
+
|
| 145 |
+
def __getstate__(self):
|
| 146 |
+
""" """
|
| 147 |
+
pass
|
| 148 |
+
|
| 149 |
+
def __setstate__(self, state):
|
| 150 |
+
""" """
|
| 151 |
+
pass
|
| 152 |
+
|
| 153 |
+
@property
|
| 154 |
+
def byte_fallback(self):
|
| 155 |
+
""" """
|
| 156 |
+
pass
|
| 157 |
+
|
| 158 |
+
@byte_fallback.setter
|
| 159 |
+
def byte_fallback(self, value):
|
| 160 |
+
""" """
|
| 161 |
+
pass
|
| 162 |
+
|
| 163 |
+
@property
|
| 164 |
+
def continuing_subword_prefix(self):
|
| 165 |
+
""" """
|
| 166 |
+
pass
|
| 167 |
+
|
| 168 |
+
@continuing_subword_prefix.setter
|
| 169 |
+
def continuing_subword_prefix(self, value):
|
| 170 |
+
""" """
|
| 171 |
+
pass
|
| 172 |
+
|
| 173 |
+
@property
|
| 174 |
+
def dropout(self):
|
| 175 |
+
""" """
|
| 176 |
+
pass
|
| 177 |
+
|
| 178 |
+
@dropout.setter
|
| 179 |
+
def dropout(self, value):
|
| 180 |
+
""" """
|
| 181 |
+
pass
|
| 182 |
+
|
| 183 |
+
@property
|
| 184 |
+
def end_of_word_suffix(self):
|
| 185 |
+
""" """
|
| 186 |
+
pass
|
| 187 |
+
|
| 188 |
+
@end_of_word_suffix.setter
|
| 189 |
+
def end_of_word_suffix(self, value):
|
| 190 |
+
""" """
|
| 191 |
+
pass
|
| 192 |
+
|
| 193 |
+
@staticmethod
|
| 194 |
+
def from_file(vocab, merges, **kwargs):
|
| 195 |
+
"""
|
| 196 |
+
Instantiate a BPE model from the given files.
|
| 197 |
+
|
| 198 |
+
This method is roughly equivalent to doing::
|
| 199 |
+
|
| 200 |
+
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
|
| 201 |
+
bpe = BPE(vocab, merges)
|
| 202 |
+
|
| 203 |
+
If you don't need to keep the :obj:`vocab, merges` values lying around,
|
| 204 |
+
this method is more optimized than manually calling
|
| 205 |
+
:meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE`
|
| 206 |
+
|
| 207 |
+
Args:
|
| 208 |
+
vocab (:obj:`str`):
|
| 209 |
+
The path to a :obj:`vocab.json` file
|
| 210 |
+
|
| 211 |
+
merges (:obj:`str`):
|
| 212 |
+
The path to a :obj:`merges.txt` file
|
| 213 |
+
|
| 214 |
+
Returns:
|
| 215 |
+
:class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files
|
| 216 |
+
"""
|
| 217 |
+
pass
|
| 218 |
+
|
| 219 |
+
@property
|
| 220 |
+
def fuse_unk(self):
|
| 221 |
+
""" """
|
| 222 |
+
pass
|
| 223 |
+
|
| 224 |
+
@fuse_unk.setter
|
| 225 |
+
def fuse_unk(self, value):
|
| 226 |
+
""" """
|
| 227 |
+
pass
|
| 228 |
+
|
| 229 |
+
def get_trainer(self):
|
| 230 |
+
"""
|
| 231 |
+
Get the associated :class:`~tokenizers.trainers.Trainer`
|
| 232 |
+
|
| 233 |
+
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
| 234 |
+
:class:`~tokenizers.models.Model`.
|
| 235 |
+
|
| 236 |
+
Returns:
|
| 237 |
+
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
| 238 |
+
"""
|
| 239 |
+
pass
|
| 240 |
+
|
| 241 |
+
def id_to_token(self, id):
|
| 242 |
+
"""
|
| 243 |
+
Get the token associated to an ID
|
| 244 |
+
|
| 245 |
+
Args:
|
| 246 |
+
id (:obj:`int`):
|
| 247 |
+
An ID to convert to a token
|
| 248 |
+
|
| 249 |
+
Returns:
|
| 250 |
+
:obj:`str`: The token associated to the ID
|
| 251 |
+
"""
|
| 252 |
+
pass
|
| 253 |
+
|
| 254 |
+
@property
|
| 255 |
+
def ignore_merges(self):
|
| 256 |
+
""" """
|
| 257 |
+
pass
|
| 258 |
+
|
| 259 |
+
@ignore_merges.setter
|
| 260 |
+
def ignore_merges(self, value):
|
| 261 |
+
""" """
|
| 262 |
+
pass
|
| 263 |
+
|
| 264 |
+
@staticmethod
|
| 265 |
+
def read_file(vocab, merges):
|
| 266 |
+
"""
|
| 267 |
+
Read a :obj:`vocab.json` and a :obj:`merges.txt` files
|
| 268 |
+
|
| 269 |
+
This method provides a way to read and parse the content of these files,
|
| 270 |
+
returning the relevant data structures. If you want to instantiate some BPE models
|
| 271 |
+
from memory, this method gives you the expected input from the standard files.
|
| 272 |
+
|
| 273 |
+
Args:
|
| 274 |
+
vocab (:obj:`str`):
|
| 275 |
+
The path to a :obj:`vocab.json` file
|
| 276 |
+
|
| 277 |
+
merges (:obj:`str`):
|
| 278 |
+
The path to a :obj:`merges.txt` file
|
| 279 |
+
|
| 280 |
+
Returns:
|
| 281 |
+
A :obj:`Tuple` with the vocab and the merges:
|
| 282 |
+
The vocabulary and merges loaded into memory
|
| 283 |
+
"""
|
| 284 |
+
pass
|
| 285 |
+
|
| 286 |
+
def save(self, folder, prefix):
|
| 287 |
+
"""
|
| 288 |
+
Save the current model
|
| 289 |
+
|
| 290 |
+
Save the current model in the given folder, using the given prefix for the various
|
| 291 |
+
files that will get created.
|
| 292 |
+
Any file with the same name that already exists in this folder will be overwritten.
|
| 293 |
+
|
| 294 |
+
Args:
|
| 295 |
+
folder (:obj:`str`):
|
| 296 |
+
The path to the target folder in which to save the various files
|
| 297 |
+
|
| 298 |
+
prefix (:obj:`str`, `optional`):
|
| 299 |
+
An optional prefix, used to prefix each file name
|
| 300 |
+
|
| 301 |
+
Returns:
|
| 302 |
+
:obj:`List[str]`: The list of saved files
|
| 303 |
+
"""
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
def token_to_id(self, tokens):
|
| 307 |
+
"""
|
| 308 |
+
Get the ID associated to a token
|
| 309 |
+
|
| 310 |
+
Args:
|
| 311 |
+
token (:obj:`str`):
|
| 312 |
+
A token to convert to an ID
|
| 313 |
+
|
| 314 |
+
Returns:
|
| 315 |
+
:obj:`int`: The ID associated to the token
|
| 316 |
+
"""
|
| 317 |
+
pass
|
| 318 |
+
|
| 319 |
+
def tokenize(self, sequence):
|
| 320 |
+
"""
|
| 321 |
+
Tokenize a sequence
|
| 322 |
+
|
| 323 |
+
Args:
|
| 324 |
+
sequence (:obj:`str`):
|
| 325 |
+
A sequence to tokenize
|
| 326 |
+
|
| 327 |
+
Returns:
|
| 328 |
+
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
| 329 |
+
"""
|
| 330 |
+
pass
|
| 331 |
+
|
| 332 |
+
@property
|
| 333 |
+
def unk_token(self):
|
| 334 |
+
""" """
|
| 335 |
+
pass
|
| 336 |
+
|
| 337 |
+
@unk_token.setter
|
| 338 |
+
def unk_token(self, value):
|
| 339 |
+
""" """
|
| 340 |
+
pass
|
| 341 |
+
|
| 342 |
+
class Unigram(Model):
|
| 343 |
+
"""
|
| 344 |
+
An implementation of the Unigram algorithm
|
| 345 |
+
|
| 346 |
+
Args:
|
| 347 |
+
vocab (:obj:`List[Tuple[str, float]]`, `optional`, `optional`):
|
| 348 |
+
A list of vocabulary items and their relative score [("am", -0.2442),...]
|
| 349 |
+
"""
|
| 350 |
+
def __init__(self, vocab=None, unk_id=None, byte_fallback=None):
|
| 351 |
+
pass
|
| 352 |
+
|
| 353 |
+
def __getstate__(self):
|
| 354 |
+
""" """
|
| 355 |
+
pass
|
| 356 |
+
|
| 357 |
+
def __setstate__(self, state):
|
| 358 |
+
""" """
|
| 359 |
+
pass
|
| 360 |
+
|
| 361 |
+
def get_trainer(self):
|
| 362 |
+
"""
|
| 363 |
+
Get the associated :class:`~tokenizers.trainers.Trainer`
|
| 364 |
+
|
| 365 |
+
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
| 366 |
+
:class:`~tokenizers.models.Model`.
|
| 367 |
+
|
| 368 |
+
Returns:
|
| 369 |
+
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
| 370 |
+
"""
|
| 371 |
+
pass
|
| 372 |
+
|
| 373 |
+
def id_to_token(self, id):
|
| 374 |
+
"""
|
| 375 |
+
Get the token associated to an ID
|
| 376 |
+
|
| 377 |
+
Args:
|
| 378 |
+
id (:obj:`int`):
|
| 379 |
+
An ID to convert to a token
|
| 380 |
+
|
| 381 |
+
Returns:
|
| 382 |
+
:obj:`str`: The token associated to the ID
|
| 383 |
+
"""
|
| 384 |
+
pass
|
| 385 |
+
|
| 386 |
+
def save(self, folder, prefix):
|
| 387 |
+
"""
|
| 388 |
+
Save the current model
|
| 389 |
+
|
| 390 |
+
Save the current model in the given folder, using the given prefix for the various
|
| 391 |
+
files that will get created.
|
| 392 |
+
Any file with the same name that already exists in this folder will be overwritten.
|
| 393 |
+
|
| 394 |
+
Args:
|
| 395 |
+
folder (:obj:`str`):
|
| 396 |
+
The path to the target folder in which to save the various files
|
| 397 |
+
|
| 398 |
+
prefix (:obj:`str`, `optional`):
|
| 399 |
+
An optional prefix, used to prefix each file name
|
| 400 |
+
|
| 401 |
+
Returns:
|
| 402 |
+
:obj:`List[str]`: The list of saved files
|
| 403 |
+
"""
|
| 404 |
+
pass
|
| 405 |
+
|
| 406 |
+
def token_to_id(self, tokens):
|
| 407 |
+
"""
|
| 408 |
+
Get the ID associated to a token
|
| 409 |
+
|
| 410 |
+
Args:
|
| 411 |
+
token (:obj:`str`):
|
| 412 |
+
A token to convert to an ID
|
| 413 |
+
|
| 414 |
+
Returns:
|
| 415 |
+
:obj:`int`: The ID associated to the token
|
| 416 |
+
"""
|
| 417 |
+
pass
|
| 418 |
+
|
| 419 |
+
def tokenize(self, sequence):
|
| 420 |
+
"""
|
| 421 |
+
Tokenize a sequence
|
| 422 |
+
|
| 423 |
+
Args:
|
| 424 |
+
sequence (:obj:`str`):
|
| 425 |
+
A sequence to tokenize
|
| 426 |
+
|
| 427 |
+
Returns:
|
| 428 |
+
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
| 429 |
+
"""
|
| 430 |
+
pass
|
| 431 |
+
|
| 432 |
+
class WordLevel(Model):
|
| 433 |
+
"""
|
| 434 |
+
An implementation of the WordLevel algorithm
|
| 435 |
+
|
| 436 |
+
Most simple tokenizer model based on mapping tokens to their corresponding id.
|
| 437 |
+
|
| 438 |
+
Args:
|
| 439 |
+
vocab (:obj:`str`, `optional`):
|
| 440 |
+
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
| 441 |
+
|
| 442 |
+
unk_token (:obj:`str`, `optional`):
|
| 443 |
+
The unknown token to be used by the model.
|
| 444 |
+
"""
|
| 445 |
+
def __init__(self, vocab=None, unk_token=None):
|
| 446 |
+
pass
|
| 447 |
+
|
| 448 |
+
def __getstate__(self):
|
| 449 |
+
""" """
|
| 450 |
+
pass
|
| 451 |
+
|
| 452 |
+
def __setstate__(self, state):
|
| 453 |
+
""" """
|
| 454 |
+
pass
|
| 455 |
+
|
| 456 |
+
@staticmethod
|
| 457 |
+
def from_file(vocab, unk_token=None):
|
| 458 |
+
"""
|
| 459 |
+
Instantiate a WordLevel model from the given file
|
| 460 |
+
|
| 461 |
+
This method is roughly equivalent to doing::
|
| 462 |
+
|
| 463 |
+
vocab = WordLevel.read_file(vocab_filename)
|
| 464 |
+
wordlevel = WordLevel(vocab)
|
| 465 |
+
|
| 466 |
+
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
| 467 |
+
more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to
|
| 468 |
+
initialize a :class:`~tokenizers.models.WordLevel`
|
| 469 |
+
|
| 470 |
+
Args:
|
| 471 |
+
vocab (:obj:`str`):
|
| 472 |
+
The path to a :obj:`vocab.json` file
|
| 473 |
+
|
| 474 |
+
Returns:
|
| 475 |
+
:class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
|
| 476 |
+
"""
|
| 477 |
+
pass
|
| 478 |
+
|
| 479 |
+
def get_trainer(self):
|
| 480 |
+
"""
|
| 481 |
+
Get the associated :class:`~tokenizers.trainers.Trainer`
|
| 482 |
+
|
| 483 |
+
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
| 484 |
+
:class:`~tokenizers.models.Model`.
|
| 485 |
+
|
| 486 |
+
Returns:
|
| 487 |
+
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
| 488 |
+
"""
|
| 489 |
+
pass
|
| 490 |
+
|
| 491 |
+
def id_to_token(self, id):
|
| 492 |
+
"""
|
| 493 |
+
Get the token associated to an ID
|
| 494 |
+
|
| 495 |
+
Args:
|
| 496 |
+
id (:obj:`int`):
|
| 497 |
+
An ID to convert to a token
|
| 498 |
+
|
| 499 |
+
Returns:
|
| 500 |
+
:obj:`str`: The token associated to the ID
|
| 501 |
+
"""
|
| 502 |
+
pass
|
| 503 |
+
|
| 504 |
+
@staticmethod
|
| 505 |
+
def read_file(vocab):
|
| 506 |
+
"""
|
| 507 |
+
Read a :obj:`vocab.json`
|
| 508 |
+
|
| 509 |
+
This method provides a way to read and parse the content of a vocabulary file,
|
| 510 |
+
returning the relevant data structures. If you want to instantiate some WordLevel models
|
| 511 |
+
from memory, this method gives you the expected input from the standard files.
|
| 512 |
+
|
| 513 |
+
Args:
|
| 514 |
+
vocab (:obj:`str`):
|
| 515 |
+
The path to a :obj:`vocab.json` file
|
| 516 |
+
|
| 517 |
+
Returns:
|
| 518 |
+
:obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
|
| 519 |
+
"""
|
| 520 |
+
pass
|
| 521 |
+
|
| 522 |
+
def save(self, folder, prefix):
|
| 523 |
+
"""
|
| 524 |
+
Save the current model
|
| 525 |
+
|
| 526 |
+
Save the current model in the given folder, using the given prefix for the various
|
| 527 |
+
files that will get created.
|
| 528 |
+
Any file with the same name that already exists in this folder will be overwritten.
|
| 529 |
+
|
| 530 |
+
Args:
|
| 531 |
+
folder (:obj:`str`):
|
| 532 |
+
The path to the target folder in which to save the various files
|
| 533 |
+
|
| 534 |
+
prefix (:obj:`str`, `optional`):
|
| 535 |
+
An optional prefix, used to prefix each file name
|
| 536 |
+
|
| 537 |
+
Returns:
|
| 538 |
+
:obj:`List[str]`: The list of saved files
|
| 539 |
+
"""
|
| 540 |
+
pass
|
| 541 |
+
|
| 542 |
+
def token_to_id(self, tokens):
|
| 543 |
+
"""
|
| 544 |
+
Get the ID associated to a token
|
| 545 |
+
|
| 546 |
+
Args:
|
| 547 |
+
token (:obj:`str`):
|
| 548 |
+
A token to convert to an ID
|
| 549 |
+
|
| 550 |
+
Returns:
|
| 551 |
+
:obj:`int`: The ID associated to the token
|
| 552 |
+
"""
|
| 553 |
+
pass
|
| 554 |
+
|
| 555 |
+
def tokenize(self, sequence):
|
| 556 |
+
"""
|
| 557 |
+
Tokenize a sequence
|
| 558 |
+
|
| 559 |
+
Args:
|
| 560 |
+
sequence (:obj:`str`):
|
| 561 |
+
A sequence to tokenize
|
| 562 |
+
|
| 563 |
+
Returns:
|
| 564 |
+
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
| 565 |
+
"""
|
| 566 |
+
pass
|
| 567 |
+
|
| 568 |
+
@property
|
| 569 |
+
def unk_token(self):
|
| 570 |
+
""" """
|
| 571 |
+
pass
|
| 572 |
+
|
| 573 |
+
@unk_token.setter
|
| 574 |
+
def unk_token(self, value):
|
| 575 |
+
""" """
|
| 576 |
+
pass
|
| 577 |
+
|
| 578 |
+
class WordPiece(Model):
|
| 579 |
+
"""
|
| 580 |
+
An implementation of the WordPiece algorithm
|
| 581 |
+
|
| 582 |
+
Args:
|
| 583 |
+
vocab (:obj:`Dict[str, int]`, `optional`):
|
| 584 |
+
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
| 585 |
+
|
| 586 |
+
unk_token (:obj:`str`, `optional`):
|
| 587 |
+
The unknown token to be used by the model.
|
| 588 |
+
|
| 589 |
+
max_input_chars_per_word (:obj:`int`, `optional`):
|
| 590 |
+
The maximum number of characters to authorize in a single word.
|
| 591 |
+
"""
|
| 592 |
+
def __init__(self, vocab=None, unk_token="[UNK]", max_input_chars_per_word=100, continuing_subword_prefix="##"):
|
| 593 |
+
pass
|
| 594 |
+
|
| 595 |
+
def __getstate__(self):
|
| 596 |
+
""" """
|
| 597 |
+
pass
|
| 598 |
+
|
| 599 |
+
def __setstate__(self, state):
|
| 600 |
+
""" """
|
| 601 |
+
pass
|
| 602 |
+
|
| 603 |
+
@property
|
| 604 |
+
def continuing_subword_prefix(self):
|
| 605 |
+
""" """
|
| 606 |
+
pass
|
| 607 |
+
|
| 608 |
+
@continuing_subword_prefix.setter
|
| 609 |
+
def continuing_subword_prefix(self, value):
|
| 610 |
+
""" """
|
| 611 |
+
pass
|
| 612 |
+
|
| 613 |
+
@staticmethod
|
| 614 |
+
def from_file(vocab, **kwargs):
|
| 615 |
+
"""
|
| 616 |
+
Instantiate a WordPiece model from the given file
|
| 617 |
+
|
| 618 |
+
This method is roughly equivalent to doing::
|
| 619 |
+
|
| 620 |
+
vocab = WordPiece.read_file(vocab_filename)
|
| 621 |
+
wordpiece = WordPiece(vocab)
|
| 622 |
+
|
| 623 |
+
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
| 624 |
+
more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to
|
| 625 |
+
initialize a :class:`~tokenizers.models.WordPiece`
|
| 626 |
+
|
| 627 |
+
Args:
|
| 628 |
+
vocab (:obj:`str`):
|
| 629 |
+
The path to a :obj:`vocab.txt` file
|
| 630 |
+
|
| 631 |
+
Returns:
|
| 632 |
+
:class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
|
| 633 |
+
"""
|
| 634 |
+
pass
|
| 635 |
+
|
| 636 |
+
def get_trainer(self):
|
| 637 |
+
"""
|
| 638 |
+
Get the associated :class:`~tokenizers.trainers.Trainer`
|
| 639 |
+
|
| 640 |
+
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
| 641 |
+
:class:`~tokenizers.models.Model`.
|
| 642 |
+
|
| 643 |
+
Returns:
|
| 644 |
+
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
| 645 |
+
"""
|
| 646 |
+
pass
|
| 647 |
+
|
| 648 |
+
def id_to_token(self, id):
|
| 649 |
+
"""
|
| 650 |
+
Get the token associated to an ID
|
| 651 |
+
|
| 652 |
+
Args:
|
| 653 |
+
id (:obj:`int`):
|
| 654 |
+
An ID to convert to a token
|
| 655 |
+
|
| 656 |
+
Returns:
|
| 657 |
+
:obj:`str`: The token associated to the ID
|
| 658 |
+
"""
|
| 659 |
+
pass
|
| 660 |
+
|
| 661 |
+
@property
|
| 662 |
+
def max_input_chars_per_word(self):
|
| 663 |
+
""" """
|
| 664 |
+
pass
|
| 665 |
+
|
| 666 |
+
@max_input_chars_per_word.setter
|
| 667 |
+
def max_input_chars_per_word(self, value):
|
| 668 |
+
""" """
|
| 669 |
+
pass
|
| 670 |
+
|
| 671 |
+
@staticmethod
|
| 672 |
+
def read_file(vocab):
|
| 673 |
+
"""
|
| 674 |
+
Read a :obj:`vocab.txt` file
|
| 675 |
+
|
| 676 |
+
This method provides a way to read and parse the content of a standard `vocab.txt`
|
| 677 |
+
file as used by the WordPiece Model, returning the relevant data structures. If you
|
| 678 |
+
want to instantiate some WordPiece models from memory, this method gives you the
|
| 679 |
+
expected input from the standard files.
|
| 680 |
+
|
| 681 |
+
Args:
|
| 682 |
+
vocab (:obj:`str`):
|
| 683 |
+
The path to a :obj:`vocab.txt` file
|
| 684 |
+
|
| 685 |
+
Returns:
|
| 686 |
+
:obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
|
| 687 |
+
"""
|
| 688 |
+
pass
|
| 689 |
+
|
| 690 |
+
def save(self, folder, prefix):
|
| 691 |
+
"""
|
| 692 |
+
Save the current model
|
| 693 |
+
|
| 694 |
+
Save the current model in the given folder, using the given prefix for the various
|
| 695 |
+
files that will get created.
|
| 696 |
+
Any file with the same name that already exists in this folder will be overwritten.
|
| 697 |
+
|
| 698 |
+
Args:
|
| 699 |
+
folder (:obj:`str`):
|
| 700 |
+
The path to the target folder in which to save the various files
|
| 701 |
+
|
| 702 |
+
prefix (:obj:`str`, `optional`):
|
| 703 |
+
An optional prefix, used to prefix each file name
|
| 704 |
+
|
| 705 |
+
Returns:
|
| 706 |
+
:obj:`List[str]`: The list of saved files
|
| 707 |
+
"""
|
| 708 |
+
pass
|
| 709 |
+
|
| 710 |
+
def token_to_id(self, tokens):
|
| 711 |
+
"""
|
| 712 |
+
Get the ID associated to a token
|
| 713 |
+
|
| 714 |
+
Args:
|
| 715 |
+
token (:obj:`str`):
|
| 716 |
+
A token to convert to an ID
|
| 717 |
+
|
| 718 |
+
Returns:
|
| 719 |
+
:obj:`int`: The ID associated to the token
|
| 720 |
+
"""
|
| 721 |
+
pass
|
| 722 |
+
|
| 723 |
+
def tokenize(self, sequence):
|
| 724 |
+
"""
|
| 725 |
+
Tokenize a sequence
|
| 726 |
+
|
| 727 |
+
Args:
|
| 728 |
+
sequence (:obj:`str`):
|
| 729 |
+
A sequence to tokenize
|
| 730 |
+
|
| 731 |
+
Returns:
|
| 732 |
+
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
| 733 |
+
"""
|
| 734 |
+
pass
|
| 735 |
+
|
| 736 |
+
@property
|
| 737 |
+
def unk_token(self):
|
| 738 |
+
""" """
|
| 739 |
+
pass
|
| 740 |
+
|
| 741 |
+
@unk_token.setter
|
| 742 |
+
def unk_token(self, value):
|
| 743 |
+
""" """
|
| 744 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/normalizers/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .. import normalizers
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Normalizer = normalizers.Normalizer
|
| 5 |
+
BertNormalizer = normalizers.BertNormalizer
|
| 6 |
+
NFD = normalizers.NFD
|
| 7 |
+
NFKD = normalizers.NFKD
|
| 8 |
+
NFC = normalizers.NFC
|
| 9 |
+
NFKC = normalizers.NFKC
|
| 10 |
+
Sequence = normalizers.Sequence
|
| 11 |
+
Lowercase = normalizers.Lowercase
|
| 12 |
+
Prepend = normalizers.Prepend
|
| 13 |
+
Strip = normalizers.Strip
|
| 14 |
+
StripAccents = normalizers.StripAccents
|
| 15 |
+
Nmt = normalizers.Nmt
|
| 16 |
+
Precompiled = normalizers.Precompiled
|
| 17 |
+
Replace = normalizers.Replace
|
| 18 |
+
ByteLevel = normalizers.ByteLevel
|
| 19 |
+
|
| 20 |
+
NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def unicode_normalizer_from_str(normalizer: str) -> Normalizer:
|
| 24 |
+
if normalizer not in NORMALIZERS:
|
| 25 |
+
raise ValueError(
|
| 26 |
+
"{} is not a known unicode normalizer. Available are {}".format(normalizer, NORMALIZERS.keys())
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
return NORMALIZERS[normalizer]()
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/normalizers/__init__.pyi
ADDED
|
@@ -0,0 +1,946 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class Normalizer:
|
| 3 |
+
"""
|
| 4 |
+
Base class for all normalizers
|
| 5 |
+
|
| 6 |
+
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
| 7 |
+
Normalizer will return an instance of this class when instantiated.
|
| 8 |
+
"""
|
| 9 |
+
def __getstate__(self):
|
| 10 |
+
""" """
|
| 11 |
+
pass
|
| 12 |
+
|
| 13 |
+
def __setstate__(self, state):
|
| 14 |
+
""" """
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
@staticmethod
|
| 18 |
+
def custom(normalizer):
|
| 19 |
+
""" """
|
| 20 |
+
pass
|
| 21 |
+
|
| 22 |
+
def normalize(self, normalized):
|
| 23 |
+
"""
|
| 24 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 25 |
+
|
| 26 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 27 |
+
keep track of the alignment information. If you just want to see the result
|
| 28 |
+
of the normalization on a raw string, you can use
|
| 29 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 30 |
+
|
| 31 |
+
Args:
|
| 32 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 33 |
+
The normalized string on which to apply this
|
| 34 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 35 |
+
"""
|
| 36 |
+
pass
|
| 37 |
+
|
| 38 |
+
def normalize_str(self, sequence):
|
| 39 |
+
"""
|
| 40 |
+
Normalize the given string
|
| 41 |
+
|
| 42 |
+
This method provides a way to visualize the effect of a
|
| 43 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 44 |
+
information. If you need to get/convert offsets, you can use
|
| 45 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
sequence (:obj:`str`):
|
| 49 |
+
A string to normalize
|
| 50 |
+
|
| 51 |
+
Returns:
|
| 52 |
+
:obj:`str`: A string after normalization
|
| 53 |
+
"""
|
| 54 |
+
pass
|
| 55 |
+
|
| 56 |
+
class BertNormalizer(Normalizer):
|
| 57 |
+
"""
|
| 58 |
+
BertNormalizer
|
| 59 |
+
|
| 60 |
+
Takes care of normalizing raw text before giving it to a Bert model.
|
| 61 |
+
This includes cleaning the text, handling accents, chinese chars and lowercasing
|
| 62 |
+
|
| 63 |
+
Args:
|
| 64 |
+
clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 65 |
+
Whether to clean the text, by removing any control characters
|
| 66 |
+
and replacing all whitespaces by the classic one.
|
| 67 |
+
|
| 68 |
+
handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 69 |
+
Whether to handle chinese chars by putting spaces around them.
|
| 70 |
+
|
| 71 |
+
strip_accents (:obj:`bool`, `optional`):
|
| 72 |
+
Whether to strip all accents. If this option is not specified (ie == None),
|
| 73 |
+
then it will be determined by the value for `lowercase` (as in the original Bert).
|
| 74 |
+
|
| 75 |
+
lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 76 |
+
Whether to lowercase.
|
| 77 |
+
"""
|
| 78 |
+
def __init__(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True):
|
| 79 |
+
pass
|
| 80 |
+
|
| 81 |
+
def __getstate__(self):
|
| 82 |
+
""" """
|
| 83 |
+
pass
|
| 84 |
+
|
| 85 |
+
def __setstate__(self, state):
|
| 86 |
+
""" """
|
| 87 |
+
pass
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def clean_text(self):
|
| 91 |
+
""" """
|
| 92 |
+
pass
|
| 93 |
+
|
| 94 |
+
@clean_text.setter
|
| 95 |
+
def clean_text(self, value):
|
| 96 |
+
""" """
|
| 97 |
+
pass
|
| 98 |
+
|
| 99 |
+
@staticmethod
|
| 100 |
+
def custom(normalizer):
|
| 101 |
+
""" """
|
| 102 |
+
pass
|
| 103 |
+
|
| 104 |
+
@property
|
| 105 |
+
def handle_chinese_chars(self):
|
| 106 |
+
""" """
|
| 107 |
+
pass
|
| 108 |
+
|
| 109 |
+
@handle_chinese_chars.setter
|
| 110 |
+
def handle_chinese_chars(self, value):
|
| 111 |
+
""" """
|
| 112 |
+
pass
|
| 113 |
+
|
| 114 |
+
@property
|
| 115 |
+
def lowercase(self):
|
| 116 |
+
""" """
|
| 117 |
+
pass
|
| 118 |
+
|
| 119 |
+
@lowercase.setter
|
| 120 |
+
def lowercase(self, value):
|
| 121 |
+
""" """
|
| 122 |
+
pass
|
| 123 |
+
|
| 124 |
+
def normalize(self, normalized):
|
| 125 |
+
"""
|
| 126 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 127 |
+
|
| 128 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 129 |
+
keep track of the alignment information. If you just want to see the result
|
| 130 |
+
of the normalization on a raw string, you can use
|
| 131 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 132 |
+
|
| 133 |
+
Args:
|
| 134 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 135 |
+
The normalized string on which to apply this
|
| 136 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 137 |
+
"""
|
| 138 |
+
pass
|
| 139 |
+
|
| 140 |
+
def normalize_str(self, sequence):
|
| 141 |
+
"""
|
| 142 |
+
Normalize the given string
|
| 143 |
+
|
| 144 |
+
This method provides a way to visualize the effect of a
|
| 145 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 146 |
+
information. If you need to get/convert offsets, you can use
|
| 147 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 148 |
+
|
| 149 |
+
Args:
|
| 150 |
+
sequence (:obj:`str`):
|
| 151 |
+
A string to normalize
|
| 152 |
+
|
| 153 |
+
Returns:
|
| 154 |
+
:obj:`str`: A string after normalization
|
| 155 |
+
"""
|
| 156 |
+
pass
|
| 157 |
+
|
| 158 |
+
@property
|
| 159 |
+
def strip_accents(self):
|
| 160 |
+
""" """
|
| 161 |
+
pass
|
| 162 |
+
|
| 163 |
+
@strip_accents.setter
|
| 164 |
+
def strip_accents(self, value):
|
| 165 |
+
""" """
|
| 166 |
+
pass
|
| 167 |
+
|
| 168 |
+
class ByteLevel(Normalizer):
|
| 169 |
+
"""
|
| 170 |
+
Bytelevel Normalizer
|
| 171 |
+
"""
|
| 172 |
+
def __init__(self):
|
| 173 |
+
pass
|
| 174 |
+
|
| 175 |
+
def __getstate__(self):
|
| 176 |
+
""" """
|
| 177 |
+
pass
|
| 178 |
+
|
| 179 |
+
def __setstate__(self, state):
|
| 180 |
+
""" """
|
| 181 |
+
pass
|
| 182 |
+
|
| 183 |
+
@staticmethod
|
| 184 |
+
def custom(normalizer):
|
| 185 |
+
""" """
|
| 186 |
+
pass
|
| 187 |
+
|
| 188 |
+
def normalize(self, normalized):
|
| 189 |
+
"""
|
| 190 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 191 |
+
|
| 192 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 193 |
+
keep track of the alignment information. If you just want to see the result
|
| 194 |
+
of the normalization on a raw string, you can use
|
| 195 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 196 |
+
|
| 197 |
+
Args:
|
| 198 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 199 |
+
The normalized string on which to apply this
|
| 200 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 201 |
+
"""
|
| 202 |
+
pass
|
| 203 |
+
|
| 204 |
+
def normalize_str(self, sequence):
|
| 205 |
+
"""
|
| 206 |
+
Normalize the given string
|
| 207 |
+
|
| 208 |
+
This method provides a way to visualize the effect of a
|
| 209 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 210 |
+
information. If you need to get/convert offsets, you can use
|
| 211 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 212 |
+
|
| 213 |
+
Args:
|
| 214 |
+
sequence (:obj:`str`):
|
| 215 |
+
A string to normalize
|
| 216 |
+
|
| 217 |
+
Returns:
|
| 218 |
+
:obj:`str`: A string after normalization
|
| 219 |
+
"""
|
| 220 |
+
pass
|
| 221 |
+
|
| 222 |
+
class Lowercase(Normalizer):
|
| 223 |
+
"""
|
| 224 |
+
Lowercase Normalizer
|
| 225 |
+
"""
|
| 226 |
+
def __init__(self):
|
| 227 |
+
pass
|
| 228 |
+
|
| 229 |
+
def __getstate__(self):
|
| 230 |
+
""" """
|
| 231 |
+
pass
|
| 232 |
+
|
| 233 |
+
def __setstate__(self, state):
|
| 234 |
+
""" """
|
| 235 |
+
pass
|
| 236 |
+
|
| 237 |
+
@staticmethod
|
| 238 |
+
def custom(normalizer):
|
| 239 |
+
""" """
|
| 240 |
+
pass
|
| 241 |
+
|
| 242 |
+
def normalize(self, normalized):
|
| 243 |
+
"""
|
| 244 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 245 |
+
|
| 246 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 247 |
+
keep track of the alignment information. If you just want to see the result
|
| 248 |
+
of the normalization on a raw string, you can use
|
| 249 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 250 |
+
|
| 251 |
+
Args:
|
| 252 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 253 |
+
The normalized string on which to apply this
|
| 254 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 255 |
+
"""
|
| 256 |
+
pass
|
| 257 |
+
|
| 258 |
+
def normalize_str(self, sequence):
|
| 259 |
+
"""
|
| 260 |
+
Normalize the given string
|
| 261 |
+
|
| 262 |
+
This method provides a way to visualize the effect of a
|
| 263 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 264 |
+
information. If you need to get/convert offsets, you can use
|
| 265 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 266 |
+
|
| 267 |
+
Args:
|
| 268 |
+
sequence (:obj:`str`):
|
| 269 |
+
A string to normalize
|
| 270 |
+
|
| 271 |
+
Returns:
|
| 272 |
+
:obj:`str`: A string after normalization
|
| 273 |
+
"""
|
| 274 |
+
pass
|
| 275 |
+
|
| 276 |
+
class NFC(Normalizer):
|
| 277 |
+
"""
|
| 278 |
+
NFC Unicode Normalizer
|
| 279 |
+
"""
|
| 280 |
+
def __init__(self):
|
| 281 |
+
pass
|
| 282 |
+
|
| 283 |
+
def __getstate__(self):
|
| 284 |
+
""" """
|
| 285 |
+
pass
|
| 286 |
+
|
| 287 |
+
def __setstate__(self, state):
|
| 288 |
+
""" """
|
| 289 |
+
pass
|
| 290 |
+
|
| 291 |
+
@staticmethod
|
| 292 |
+
def custom(normalizer):
|
| 293 |
+
""" """
|
| 294 |
+
pass
|
| 295 |
+
|
| 296 |
+
def normalize(self, normalized):
|
| 297 |
+
"""
|
| 298 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 299 |
+
|
| 300 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 301 |
+
keep track of the alignment information. If you just want to see the result
|
| 302 |
+
of the normalization on a raw string, you can use
|
| 303 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 304 |
+
|
| 305 |
+
Args:
|
| 306 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 307 |
+
The normalized string on which to apply this
|
| 308 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 309 |
+
"""
|
| 310 |
+
pass
|
| 311 |
+
|
| 312 |
+
def normalize_str(self, sequence):
|
| 313 |
+
"""
|
| 314 |
+
Normalize the given string
|
| 315 |
+
|
| 316 |
+
This method provides a way to visualize the effect of a
|
| 317 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 318 |
+
information. If you need to get/convert offsets, you can use
|
| 319 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 320 |
+
|
| 321 |
+
Args:
|
| 322 |
+
sequence (:obj:`str`):
|
| 323 |
+
A string to normalize
|
| 324 |
+
|
| 325 |
+
Returns:
|
| 326 |
+
:obj:`str`: A string after normalization
|
| 327 |
+
"""
|
| 328 |
+
pass
|
| 329 |
+
|
| 330 |
+
class NFD(Normalizer):
|
| 331 |
+
"""
|
| 332 |
+
NFD Unicode Normalizer
|
| 333 |
+
"""
|
| 334 |
+
def __init__(self):
|
| 335 |
+
pass
|
| 336 |
+
|
| 337 |
+
def __getstate__(self):
|
| 338 |
+
""" """
|
| 339 |
+
pass
|
| 340 |
+
|
| 341 |
+
def __setstate__(self, state):
|
| 342 |
+
""" """
|
| 343 |
+
pass
|
| 344 |
+
|
| 345 |
+
@staticmethod
|
| 346 |
+
def custom(normalizer):
|
| 347 |
+
""" """
|
| 348 |
+
pass
|
| 349 |
+
|
| 350 |
+
def normalize(self, normalized):
|
| 351 |
+
"""
|
| 352 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 353 |
+
|
| 354 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 355 |
+
keep track of the alignment information. If you just want to see the result
|
| 356 |
+
of the normalization on a raw string, you can use
|
| 357 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 358 |
+
|
| 359 |
+
Args:
|
| 360 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 361 |
+
The normalized string on which to apply this
|
| 362 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 363 |
+
"""
|
| 364 |
+
pass
|
| 365 |
+
|
| 366 |
+
def normalize_str(self, sequence):
|
| 367 |
+
"""
|
| 368 |
+
Normalize the given string
|
| 369 |
+
|
| 370 |
+
This method provides a way to visualize the effect of a
|
| 371 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 372 |
+
information. If you need to get/convert offsets, you can use
|
| 373 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 374 |
+
|
| 375 |
+
Args:
|
| 376 |
+
sequence (:obj:`str`):
|
| 377 |
+
A string to normalize
|
| 378 |
+
|
| 379 |
+
Returns:
|
| 380 |
+
:obj:`str`: A string after normalization
|
| 381 |
+
"""
|
| 382 |
+
pass
|
| 383 |
+
|
| 384 |
+
class NFKC(Normalizer):
|
| 385 |
+
"""
|
| 386 |
+
NFKC Unicode Normalizer
|
| 387 |
+
"""
|
| 388 |
+
def __init__(self):
|
| 389 |
+
pass
|
| 390 |
+
|
| 391 |
+
def __getstate__(self):
|
| 392 |
+
""" """
|
| 393 |
+
pass
|
| 394 |
+
|
| 395 |
+
def __setstate__(self, state):
|
| 396 |
+
""" """
|
| 397 |
+
pass
|
| 398 |
+
|
| 399 |
+
@staticmethod
|
| 400 |
+
def custom(normalizer):
|
| 401 |
+
""" """
|
| 402 |
+
pass
|
| 403 |
+
|
| 404 |
+
def normalize(self, normalized):
|
| 405 |
+
"""
|
| 406 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 407 |
+
|
| 408 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 409 |
+
keep track of the alignment information. If you just want to see the result
|
| 410 |
+
of the normalization on a raw string, you can use
|
| 411 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 412 |
+
|
| 413 |
+
Args:
|
| 414 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 415 |
+
The normalized string on which to apply this
|
| 416 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 417 |
+
"""
|
| 418 |
+
pass
|
| 419 |
+
|
| 420 |
+
def normalize_str(self, sequence):
|
| 421 |
+
"""
|
| 422 |
+
Normalize the given string
|
| 423 |
+
|
| 424 |
+
This method provides a way to visualize the effect of a
|
| 425 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 426 |
+
information. If you need to get/convert offsets, you can use
|
| 427 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 428 |
+
|
| 429 |
+
Args:
|
| 430 |
+
sequence (:obj:`str`):
|
| 431 |
+
A string to normalize
|
| 432 |
+
|
| 433 |
+
Returns:
|
| 434 |
+
:obj:`str`: A string after normalization
|
| 435 |
+
"""
|
| 436 |
+
pass
|
| 437 |
+
|
| 438 |
+
class NFKD(Normalizer):
|
| 439 |
+
"""
|
| 440 |
+
NFKD Unicode Normalizer
|
| 441 |
+
"""
|
| 442 |
+
def __init__(self):
|
| 443 |
+
pass
|
| 444 |
+
|
| 445 |
+
def __getstate__(self):
|
| 446 |
+
""" """
|
| 447 |
+
pass
|
| 448 |
+
|
| 449 |
+
def __setstate__(self, state):
|
| 450 |
+
""" """
|
| 451 |
+
pass
|
| 452 |
+
|
| 453 |
+
@staticmethod
|
| 454 |
+
def custom(normalizer):
|
| 455 |
+
""" """
|
| 456 |
+
pass
|
| 457 |
+
|
| 458 |
+
def normalize(self, normalized):
|
| 459 |
+
"""
|
| 460 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 461 |
+
|
| 462 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 463 |
+
keep track of the alignment information. If you just want to see the result
|
| 464 |
+
of the normalization on a raw string, you can use
|
| 465 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 466 |
+
|
| 467 |
+
Args:
|
| 468 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 469 |
+
The normalized string on which to apply this
|
| 470 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 471 |
+
"""
|
| 472 |
+
pass
|
| 473 |
+
|
| 474 |
+
def normalize_str(self, sequence):
|
| 475 |
+
"""
|
| 476 |
+
Normalize the given string
|
| 477 |
+
|
| 478 |
+
This method provides a way to visualize the effect of a
|
| 479 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 480 |
+
information. If you need to get/convert offsets, you can use
|
| 481 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 482 |
+
|
| 483 |
+
Args:
|
| 484 |
+
sequence (:obj:`str`):
|
| 485 |
+
A string to normalize
|
| 486 |
+
|
| 487 |
+
Returns:
|
| 488 |
+
:obj:`str`: A string after normalization
|
| 489 |
+
"""
|
| 490 |
+
pass
|
| 491 |
+
|
| 492 |
+
class Nmt(Normalizer):
|
| 493 |
+
"""
|
| 494 |
+
Nmt normalizer
|
| 495 |
+
"""
|
| 496 |
+
def __init__(self):
|
| 497 |
+
pass
|
| 498 |
+
|
| 499 |
+
def __getstate__(self):
|
| 500 |
+
""" """
|
| 501 |
+
pass
|
| 502 |
+
|
| 503 |
+
def __setstate__(self, state):
|
| 504 |
+
""" """
|
| 505 |
+
pass
|
| 506 |
+
|
| 507 |
+
@staticmethod
|
| 508 |
+
def custom(normalizer):
|
| 509 |
+
""" """
|
| 510 |
+
pass
|
| 511 |
+
|
| 512 |
+
def normalize(self, normalized):
|
| 513 |
+
"""
|
| 514 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 515 |
+
|
| 516 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 517 |
+
keep track of the alignment information. If you just want to see the result
|
| 518 |
+
of the normalization on a raw string, you can use
|
| 519 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 520 |
+
|
| 521 |
+
Args:
|
| 522 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 523 |
+
The normalized string on which to apply this
|
| 524 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 525 |
+
"""
|
| 526 |
+
pass
|
| 527 |
+
|
| 528 |
+
def normalize_str(self, sequence):
|
| 529 |
+
"""
|
| 530 |
+
Normalize the given string
|
| 531 |
+
|
| 532 |
+
This method provides a way to visualize the effect of a
|
| 533 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 534 |
+
information. If you need to get/convert offsets, you can use
|
| 535 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 536 |
+
|
| 537 |
+
Args:
|
| 538 |
+
sequence (:obj:`str`):
|
| 539 |
+
A string to normalize
|
| 540 |
+
|
| 541 |
+
Returns:
|
| 542 |
+
:obj:`str`: A string after normalization
|
| 543 |
+
"""
|
| 544 |
+
pass
|
| 545 |
+
|
| 546 |
+
class Precompiled(Normalizer):
|
| 547 |
+
"""
|
| 548 |
+
Precompiled normalizer
|
| 549 |
+
Don't use manually it is used for compatibility for SentencePiece.
|
| 550 |
+
"""
|
| 551 |
+
def __init__(self, precompiled_charsmap):
|
| 552 |
+
pass
|
| 553 |
+
|
| 554 |
+
def __getstate__(self):
|
| 555 |
+
""" """
|
| 556 |
+
pass
|
| 557 |
+
|
| 558 |
+
def __setstate__(self, state):
|
| 559 |
+
""" """
|
| 560 |
+
pass
|
| 561 |
+
|
| 562 |
+
@staticmethod
|
| 563 |
+
def custom(normalizer):
|
| 564 |
+
""" """
|
| 565 |
+
pass
|
| 566 |
+
|
| 567 |
+
def normalize(self, normalized):
|
| 568 |
+
"""
|
| 569 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 570 |
+
|
| 571 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 572 |
+
keep track of the alignment information. If you just want to see the result
|
| 573 |
+
of the normalization on a raw string, you can use
|
| 574 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 575 |
+
|
| 576 |
+
Args:
|
| 577 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 578 |
+
The normalized string on which to apply this
|
| 579 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 580 |
+
"""
|
| 581 |
+
pass
|
| 582 |
+
|
| 583 |
+
def normalize_str(self, sequence):
|
| 584 |
+
"""
|
| 585 |
+
Normalize the given string
|
| 586 |
+
|
| 587 |
+
This method provides a way to visualize the effect of a
|
| 588 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 589 |
+
information. If you need to get/convert offsets, you can use
|
| 590 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 591 |
+
|
| 592 |
+
Args:
|
| 593 |
+
sequence (:obj:`str`):
|
| 594 |
+
A string to normalize
|
| 595 |
+
|
| 596 |
+
Returns:
|
| 597 |
+
:obj:`str`: A string after normalization
|
| 598 |
+
"""
|
| 599 |
+
pass
|
| 600 |
+
|
| 601 |
+
class Prepend(Normalizer):
|
| 602 |
+
"""
|
| 603 |
+
Prepend normalizer
|
| 604 |
+
"""
|
| 605 |
+
def __init__(self, prepend):
|
| 606 |
+
pass
|
| 607 |
+
|
| 608 |
+
def __getstate__(self):
|
| 609 |
+
""" """
|
| 610 |
+
pass
|
| 611 |
+
|
| 612 |
+
def __setstate__(self, state):
|
| 613 |
+
""" """
|
| 614 |
+
pass
|
| 615 |
+
|
| 616 |
+
@staticmethod
|
| 617 |
+
def custom(normalizer):
|
| 618 |
+
""" """
|
| 619 |
+
pass
|
| 620 |
+
|
| 621 |
+
def normalize(self, normalized):
|
| 622 |
+
"""
|
| 623 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 624 |
+
|
| 625 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 626 |
+
keep track of the alignment information. If you just want to see the result
|
| 627 |
+
of the normalization on a raw string, you can use
|
| 628 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 629 |
+
|
| 630 |
+
Args:
|
| 631 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 632 |
+
The normalized string on which to apply this
|
| 633 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 634 |
+
"""
|
| 635 |
+
pass
|
| 636 |
+
|
| 637 |
+
def normalize_str(self, sequence):
|
| 638 |
+
"""
|
| 639 |
+
Normalize the given string
|
| 640 |
+
|
| 641 |
+
This method provides a way to visualize the effect of a
|
| 642 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 643 |
+
information. If you need to get/convert offsets, you can use
|
| 644 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 645 |
+
|
| 646 |
+
Args:
|
| 647 |
+
sequence (:obj:`str`):
|
| 648 |
+
A string to normalize
|
| 649 |
+
|
| 650 |
+
Returns:
|
| 651 |
+
:obj:`str`: A string after normalization
|
| 652 |
+
"""
|
| 653 |
+
pass
|
| 654 |
+
|
| 655 |
+
@property
|
| 656 |
+
def prepend(self):
|
| 657 |
+
""" """
|
| 658 |
+
pass
|
| 659 |
+
|
| 660 |
+
@prepend.setter
|
| 661 |
+
def prepend(self, value):
|
| 662 |
+
""" """
|
| 663 |
+
pass
|
| 664 |
+
|
| 665 |
+
class Replace(Normalizer):
|
| 666 |
+
"""
|
| 667 |
+
Replace normalizer
|
| 668 |
+
"""
|
| 669 |
+
def __init__(self, pattern, content):
|
| 670 |
+
pass
|
| 671 |
+
|
| 672 |
+
def __getstate__(self):
|
| 673 |
+
""" """
|
| 674 |
+
pass
|
| 675 |
+
|
| 676 |
+
def __setstate__(self, state):
|
| 677 |
+
""" """
|
| 678 |
+
pass
|
| 679 |
+
|
| 680 |
+
@property
|
| 681 |
+
def content(self):
|
| 682 |
+
""" """
|
| 683 |
+
pass
|
| 684 |
+
|
| 685 |
+
@content.setter
|
| 686 |
+
def content(self, value):
|
| 687 |
+
""" """
|
| 688 |
+
pass
|
| 689 |
+
|
| 690 |
+
@staticmethod
|
| 691 |
+
def custom(normalizer):
|
| 692 |
+
""" """
|
| 693 |
+
pass
|
| 694 |
+
|
| 695 |
+
def normalize(self, normalized):
|
| 696 |
+
"""
|
| 697 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 698 |
+
|
| 699 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 700 |
+
keep track of the alignment information. If you just want to see the result
|
| 701 |
+
of the normalization on a raw string, you can use
|
| 702 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 703 |
+
|
| 704 |
+
Args:
|
| 705 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 706 |
+
The normalized string on which to apply this
|
| 707 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 708 |
+
"""
|
| 709 |
+
pass
|
| 710 |
+
|
| 711 |
+
def normalize_str(self, sequence):
|
| 712 |
+
"""
|
| 713 |
+
Normalize the given string
|
| 714 |
+
|
| 715 |
+
This method provides a way to visualize the effect of a
|
| 716 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 717 |
+
information. If you need to get/convert offsets, you can use
|
| 718 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 719 |
+
|
| 720 |
+
Args:
|
| 721 |
+
sequence (:obj:`str`):
|
| 722 |
+
A string to normalize
|
| 723 |
+
|
| 724 |
+
Returns:
|
| 725 |
+
:obj:`str`: A string after normalization
|
| 726 |
+
"""
|
| 727 |
+
pass
|
| 728 |
+
|
| 729 |
+
@property
|
| 730 |
+
def pattern(self):
|
| 731 |
+
""" """
|
| 732 |
+
pass
|
| 733 |
+
|
| 734 |
+
@pattern.setter
|
| 735 |
+
def pattern(self, value):
|
| 736 |
+
""" """
|
| 737 |
+
pass
|
| 738 |
+
|
| 739 |
+
class Sequence(Normalizer):
|
| 740 |
+
"""
|
| 741 |
+
Allows concatenating multiple other Normalizer as a Sequence.
|
| 742 |
+
All the normalizers run in sequence in the given order
|
| 743 |
+
|
| 744 |
+
Args:
|
| 745 |
+
normalizers (:obj:`List[Normalizer]`):
|
| 746 |
+
A list of Normalizer to be run as a sequence
|
| 747 |
+
"""
|
| 748 |
+
def __init__(self, normalizers):
|
| 749 |
+
pass
|
| 750 |
+
|
| 751 |
+
def __getitem__(self, key):
|
| 752 |
+
"""
|
| 753 |
+
Return self[key].
|
| 754 |
+
"""
|
| 755 |
+
pass
|
| 756 |
+
|
| 757 |
+
def __getnewargs__(self):
|
| 758 |
+
""" """
|
| 759 |
+
pass
|
| 760 |
+
|
| 761 |
+
def __getstate__(self):
|
| 762 |
+
""" """
|
| 763 |
+
pass
|
| 764 |
+
|
| 765 |
+
def __setitem__(self, key, value):
|
| 766 |
+
"""
|
| 767 |
+
Set self[key] to value.
|
| 768 |
+
"""
|
| 769 |
+
pass
|
| 770 |
+
|
| 771 |
+
def __setstate__(self, state):
|
| 772 |
+
""" """
|
| 773 |
+
pass
|
| 774 |
+
|
| 775 |
+
@staticmethod
|
| 776 |
+
def custom(normalizer):
|
| 777 |
+
""" """
|
| 778 |
+
pass
|
| 779 |
+
|
| 780 |
+
def normalize(self, normalized):
|
| 781 |
+
"""
|
| 782 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 783 |
+
|
| 784 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 785 |
+
keep track of the alignment information. If you just want to see the result
|
| 786 |
+
of the normalization on a raw string, you can use
|
| 787 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 788 |
+
|
| 789 |
+
Args:
|
| 790 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 791 |
+
The normalized string on which to apply this
|
| 792 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 793 |
+
"""
|
| 794 |
+
pass
|
| 795 |
+
|
| 796 |
+
def normalize_str(self, sequence):
|
| 797 |
+
"""
|
| 798 |
+
Normalize the given string
|
| 799 |
+
|
| 800 |
+
This method provides a way to visualize the effect of a
|
| 801 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 802 |
+
information. If you need to get/convert offsets, you can use
|
| 803 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 804 |
+
|
| 805 |
+
Args:
|
| 806 |
+
sequence (:obj:`str`):
|
| 807 |
+
A string to normalize
|
| 808 |
+
|
| 809 |
+
Returns:
|
| 810 |
+
:obj:`str`: A string after normalization
|
| 811 |
+
"""
|
| 812 |
+
pass
|
| 813 |
+
|
| 814 |
+
class Strip(Normalizer):
|
| 815 |
+
"""
|
| 816 |
+
Strip normalizer
|
| 817 |
+
"""
|
| 818 |
+
def __init__(self, left=True, right=True):
|
| 819 |
+
pass
|
| 820 |
+
|
| 821 |
+
def __getstate__(self):
|
| 822 |
+
""" """
|
| 823 |
+
pass
|
| 824 |
+
|
| 825 |
+
def __setstate__(self, state):
|
| 826 |
+
""" """
|
| 827 |
+
pass
|
| 828 |
+
|
| 829 |
+
@staticmethod
|
| 830 |
+
def custom(normalizer):
|
| 831 |
+
""" """
|
| 832 |
+
pass
|
| 833 |
+
|
| 834 |
+
@property
|
| 835 |
+
def left(self):
|
| 836 |
+
""" """
|
| 837 |
+
pass
|
| 838 |
+
|
| 839 |
+
@left.setter
|
| 840 |
+
def left(self, value):
|
| 841 |
+
""" """
|
| 842 |
+
pass
|
| 843 |
+
|
| 844 |
+
def normalize(self, normalized):
|
| 845 |
+
"""
|
| 846 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 847 |
+
|
| 848 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 849 |
+
keep track of the alignment information. If you just want to see the result
|
| 850 |
+
of the normalization on a raw string, you can use
|
| 851 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 852 |
+
|
| 853 |
+
Args:
|
| 854 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 855 |
+
The normalized string on which to apply this
|
| 856 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 857 |
+
"""
|
| 858 |
+
pass
|
| 859 |
+
|
| 860 |
+
def normalize_str(self, sequence):
|
| 861 |
+
"""
|
| 862 |
+
Normalize the given string
|
| 863 |
+
|
| 864 |
+
This method provides a way to visualize the effect of a
|
| 865 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 866 |
+
information. If you need to get/convert offsets, you can use
|
| 867 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 868 |
+
|
| 869 |
+
Args:
|
| 870 |
+
sequence (:obj:`str`):
|
| 871 |
+
A string to normalize
|
| 872 |
+
|
| 873 |
+
Returns:
|
| 874 |
+
:obj:`str`: A string after normalization
|
| 875 |
+
"""
|
| 876 |
+
pass
|
| 877 |
+
|
| 878 |
+
@property
|
| 879 |
+
def right(self):
|
| 880 |
+
""" """
|
| 881 |
+
pass
|
| 882 |
+
|
| 883 |
+
@right.setter
|
| 884 |
+
def right(self, value):
|
| 885 |
+
""" """
|
| 886 |
+
pass
|
| 887 |
+
|
| 888 |
+
class StripAccents(Normalizer):
|
| 889 |
+
"""
|
| 890 |
+
StripAccents normalizer
|
| 891 |
+
"""
|
| 892 |
+
def __init__(self):
|
| 893 |
+
pass
|
| 894 |
+
|
| 895 |
+
def __getstate__(self):
|
| 896 |
+
""" """
|
| 897 |
+
pass
|
| 898 |
+
|
| 899 |
+
def __setstate__(self, state):
|
| 900 |
+
""" """
|
| 901 |
+
pass
|
| 902 |
+
|
| 903 |
+
@staticmethod
|
| 904 |
+
def custom(normalizer):
|
| 905 |
+
""" """
|
| 906 |
+
pass
|
| 907 |
+
|
| 908 |
+
def normalize(self, normalized):
|
| 909 |
+
"""
|
| 910 |
+
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
| 911 |
+
|
| 912 |
+
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
| 913 |
+
keep track of the alignment information. If you just want to see the result
|
| 914 |
+
of the normalization on a raw string, you can use
|
| 915 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
| 916 |
+
|
| 917 |
+
Args:
|
| 918 |
+
normalized (:class:`~tokenizers.NormalizedString`):
|
| 919 |
+
The normalized string on which to apply this
|
| 920 |
+
:class:`~tokenizers.normalizers.Normalizer`
|
| 921 |
+
"""
|
| 922 |
+
pass
|
| 923 |
+
|
| 924 |
+
def normalize_str(self, sequence):
|
| 925 |
+
"""
|
| 926 |
+
Normalize the given string
|
| 927 |
+
|
| 928 |
+
This method provides a way to visualize the effect of a
|
| 929 |
+
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
| 930 |
+
information. If you need to get/convert offsets, you can use
|
| 931 |
+
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
| 932 |
+
|
| 933 |
+
Args:
|
| 934 |
+
sequence (:obj:`str`):
|
| 935 |
+
A string to normalize
|
| 936 |
+
|
| 937 |
+
Returns:
|
| 938 |
+
:obj:`str`: A string after normalization
|
| 939 |
+
"""
|
| 940 |
+
pass
|
| 941 |
+
|
| 942 |
+
from typing import Dict
|
| 943 |
+
|
| 944 |
+
NORMALIZERS: Dict[str, Normalizer]
|
| 945 |
+
|
| 946 |
+
def unicode_normalizer_from_str(normalizer: str) -> Normalizer: ...
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
from .. import pre_tokenizers
|
| 3 |
+
|
| 4 |
+
PreTokenizer = pre_tokenizers.PreTokenizer
|
| 5 |
+
BertPreTokenizer = pre_tokenizers.BertPreTokenizer
|
| 6 |
+
ByteLevel = pre_tokenizers.ByteLevel
|
| 7 |
+
CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit
|
| 8 |
+
Digits = pre_tokenizers.Digits
|
| 9 |
+
FixedLength = pre_tokenizers.FixedLength
|
| 10 |
+
Metaspace = pre_tokenizers.Metaspace
|
| 11 |
+
Punctuation = pre_tokenizers.Punctuation
|
| 12 |
+
Sequence = pre_tokenizers.Sequence
|
| 13 |
+
Split = pre_tokenizers.Split
|
| 14 |
+
UnicodeScripts = pre_tokenizers.UnicodeScripts
|
| 15 |
+
Whitespace = pre_tokenizers.Whitespace
|
| 16 |
+
WhitespaceSplit = pre_tokenizers.WhitespaceSplit
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.pyi
ADDED
|
@@ -0,0 +1,1015 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class PreTokenizer:
|
| 3 |
+
"""
|
| 4 |
+
Base class for all pre-tokenizers
|
| 5 |
+
|
| 6 |
+
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
| 7 |
+
PreTokenizer will return an instance of this class when instantiated.
|
| 8 |
+
"""
|
| 9 |
+
def __getstate__(self):
|
| 10 |
+
""" """
|
| 11 |
+
pass
|
| 12 |
+
|
| 13 |
+
def __setstate__(self, state):
|
| 14 |
+
""" """
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
@staticmethod
|
| 18 |
+
def custom(pretok):
|
| 19 |
+
""" """
|
| 20 |
+
pass
|
| 21 |
+
|
| 22 |
+
def pre_tokenize(self, pretok):
|
| 23 |
+
"""
|
| 24 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 25 |
+
|
| 26 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 27 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 28 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 29 |
+
the pre-tokenization of a raw string, you can use
|
| 30 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 34 |
+
The pre-tokenized string on which to apply this
|
| 35 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 36 |
+
"""
|
| 37 |
+
pass
|
| 38 |
+
|
| 39 |
+
def pre_tokenize_str(self, sequence):
|
| 40 |
+
"""
|
| 41 |
+
Pre tokenize the given string
|
| 42 |
+
|
| 43 |
+
This method provides a way to visualize the effect of a
|
| 44 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 45 |
+
alignment, nor does it provide all the capabilities of the
|
| 46 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 47 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 48 |
+
|
| 49 |
+
Args:
|
| 50 |
+
sequence (:obj:`str`):
|
| 51 |
+
A string to pre-tokeize
|
| 52 |
+
|
| 53 |
+
Returns:
|
| 54 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 55 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 56 |
+
"""
|
| 57 |
+
pass
|
| 58 |
+
|
| 59 |
+
class BertPreTokenizer(PreTokenizer):
|
| 60 |
+
"""
|
| 61 |
+
BertPreTokenizer
|
| 62 |
+
|
| 63 |
+
This pre-tokenizer splits tokens on spaces, and also on punctuation.
|
| 64 |
+
Each occurrence of a punctuation character will be treated separately.
|
| 65 |
+
"""
|
| 66 |
+
def __init__(self):
|
| 67 |
+
pass
|
| 68 |
+
|
| 69 |
+
def __getstate__(self):
|
| 70 |
+
""" """
|
| 71 |
+
pass
|
| 72 |
+
|
| 73 |
+
def __setstate__(self, state):
|
| 74 |
+
""" """
|
| 75 |
+
pass
|
| 76 |
+
|
| 77 |
+
@staticmethod
|
| 78 |
+
def custom(pretok):
|
| 79 |
+
""" """
|
| 80 |
+
pass
|
| 81 |
+
|
| 82 |
+
def pre_tokenize(self, pretok):
|
| 83 |
+
"""
|
| 84 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 85 |
+
|
| 86 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 87 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 88 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 89 |
+
the pre-tokenization of a raw string, you can use
|
| 90 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 91 |
+
|
| 92 |
+
Args:
|
| 93 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 94 |
+
The pre-tokenized string on which to apply this
|
| 95 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 96 |
+
"""
|
| 97 |
+
pass
|
| 98 |
+
|
| 99 |
+
def pre_tokenize_str(self, sequence):
|
| 100 |
+
"""
|
| 101 |
+
Pre tokenize the given string
|
| 102 |
+
|
| 103 |
+
This method provides a way to visualize the effect of a
|
| 104 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 105 |
+
alignment, nor does it provide all the capabilities of the
|
| 106 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 107 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 108 |
+
|
| 109 |
+
Args:
|
| 110 |
+
sequence (:obj:`str`):
|
| 111 |
+
A string to pre-tokeize
|
| 112 |
+
|
| 113 |
+
Returns:
|
| 114 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 115 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 116 |
+
"""
|
| 117 |
+
pass
|
| 118 |
+
|
| 119 |
+
class ByteLevel(PreTokenizer):
|
| 120 |
+
"""
|
| 121 |
+
ByteLevel PreTokenizer
|
| 122 |
+
|
| 123 |
+
This pre-tokenizer takes care of replacing all bytes of the given string
|
| 124 |
+
with a corresponding representation, as well as splitting into words.
|
| 125 |
+
|
| 126 |
+
Args:
|
| 127 |
+
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 128 |
+
Whether to add a space to the first word if there isn't already one. This
|
| 129 |
+
lets us treat `hello` exactly like `say hello`.
|
| 130 |
+
use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 131 |
+
Set this to :obj:`False` to prevent this `pre_tokenizer` from using
|
| 132 |
+
the GPT2 specific regexp for spliting on whitespace.
|
| 133 |
+
"""
|
| 134 |
+
def __init__(self, add_prefix_space=True, trim_offsets=True, use_regex=True):
|
| 135 |
+
pass
|
| 136 |
+
|
| 137 |
+
def __getstate__(self):
|
| 138 |
+
""" """
|
| 139 |
+
pass
|
| 140 |
+
|
| 141 |
+
def __setstate__(self, state):
|
| 142 |
+
""" """
|
| 143 |
+
pass
|
| 144 |
+
|
| 145 |
+
@property
|
| 146 |
+
def add_prefix_space(self):
|
| 147 |
+
""" """
|
| 148 |
+
pass
|
| 149 |
+
|
| 150 |
+
@add_prefix_space.setter
|
| 151 |
+
def add_prefix_space(self, value):
|
| 152 |
+
""" """
|
| 153 |
+
pass
|
| 154 |
+
|
| 155 |
+
@staticmethod
|
| 156 |
+
def alphabet():
|
| 157 |
+
"""
|
| 158 |
+
Returns the alphabet used by this PreTokenizer.
|
| 159 |
+
|
| 160 |
+
Since the ByteLevel works as its name suggests, at the byte level, it
|
| 161 |
+
encodes each byte value to a unique visible character. This means that there is a
|
| 162 |
+
total of 256 different characters composing this alphabet.
|
| 163 |
+
|
| 164 |
+
Returns:
|
| 165 |
+
:obj:`List[str]`: A list of characters that compose the alphabet
|
| 166 |
+
"""
|
| 167 |
+
pass
|
| 168 |
+
|
| 169 |
+
@staticmethod
|
| 170 |
+
def custom(pretok):
|
| 171 |
+
""" """
|
| 172 |
+
pass
|
| 173 |
+
|
| 174 |
+
def pre_tokenize(self, pretok):
|
| 175 |
+
"""
|
| 176 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 177 |
+
|
| 178 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 179 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 180 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 181 |
+
the pre-tokenization of a raw string, you can use
|
| 182 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 183 |
+
|
| 184 |
+
Args:
|
| 185 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 186 |
+
The pre-tokenized string on which to apply this
|
| 187 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 188 |
+
"""
|
| 189 |
+
pass
|
| 190 |
+
|
| 191 |
+
def pre_tokenize_str(self, sequence):
|
| 192 |
+
"""
|
| 193 |
+
Pre tokenize the given string
|
| 194 |
+
|
| 195 |
+
This method provides a way to visualize the effect of a
|
| 196 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 197 |
+
alignment, nor does it provide all the capabilities of the
|
| 198 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 199 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 200 |
+
|
| 201 |
+
Args:
|
| 202 |
+
sequence (:obj:`str`):
|
| 203 |
+
A string to pre-tokeize
|
| 204 |
+
|
| 205 |
+
Returns:
|
| 206 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 207 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 208 |
+
"""
|
| 209 |
+
pass
|
| 210 |
+
|
| 211 |
+
@property
|
| 212 |
+
def trim_offsets(self):
|
| 213 |
+
""" """
|
| 214 |
+
pass
|
| 215 |
+
|
| 216 |
+
@trim_offsets.setter
|
| 217 |
+
def trim_offsets(self, value):
|
| 218 |
+
""" """
|
| 219 |
+
pass
|
| 220 |
+
|
| 221 |
+
@property
|
| 222 |
+
def use_regex(self):
|
| 223 |
+
""" """
|
| 224 |
+
pass
|
| 225 |
+
|
| 226 |
+
@use_regex.setter
|
| 227 |
+
def use_regex(self, value):
|
| 228 |
+
""" """
|
| 229 |
+
pass
|
| 230 |
+
|
| 231 |
+
class CharDelimiterSplit(PreTokenizer):
|
| 232 |
+
"""
|
| 233 |
+
This pre-tokenizer simply splits on the provided char. Works like `.split(delimiter)`
|
| 234 |
+
|
| 235 |
+
Args:
|
| 236 |
+
delimiter: str:
|
| 237 |
+
The delimiter char that will be used to split input
|
| 238 |
+
"""
|
| 239 |
+
def __init__(self, delimiter):
|
| 240 |
+
pass
|
| 241 |
+
|
| 242 |
+
def __getnewargs__(self):
|
| 243 |
+
""" """
|
| 244 |
+
pass
|
| 245 |
+
|
| 246 |
+
def __getstate__(self):
|
| 247 |
+
""" """
|
| 248 |
+
pass
|
| 249 |
+
|
| 250 |
+
def __setstate__(self, state):
|
| 251 |
+
""" """
|
| 252 |
+
pass
|
| 253 |
+
|
| 254 |
+
@staticmethod
|
| 255 |
+
def custom(pretok):
|
| 256 |
+
""" """
|
| 257 |
+
pass
|
| 258 |
+
|
| 259 |
+
@property
|
| 260 |
+
def delimiter(self):
|
| 261 |
+
""" """
|
| 262 |
+
pass
|
| 263 |
+
|
| 264 |
+
@delimiter.setter
|
| 265 |
+
def delimiter(self, value):
|
| 266 |
+
""" """
|
| 267 |
+
pass
|
| 268 |
+
|
| 269 |
+
def pre_tokenize(self, pretok):
|
| 270 |
+
"""
|
| 271 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 272 |
+
|
| 273 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 274 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 275 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 276 |
+
the pre-tokenization of a raw string, you can use
|
| 277 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 278 |
+
|
| 279 |
+
Args:
|
| 280 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 281 |
+
The pre-tokenized string on which to apply this
|
| 282 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 283 |
+
"""
|
| 284 |
+
pass
|
| 285 |
+
|
| 286 |
+
def pre_tokenize_str(self, sequence):
|
| 287 |
+
"""
|
| 288 |
+
Pre tokenize the given string
|
| 289 |
+
|
| 290 |
+
This method provides a way to visualize the effect of a
|
| 291 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 292 |
+
alignment, nor does it provide all the capabilities of the
|
| 293 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 294 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 295 |
+
|
| 296 |
+
Args:
|
| 297 |
+
sequence (:obj:`str`):
|
| 298 |
+
A string to pre-tokeize
|
| 299 |
+
|
| 300 |
+
Returns:
|
| 301 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 302 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 303 |
+
"""
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
class Digits(PreTokenizer):
|
| 307 |
+
"""
|
| 308 |
+
This pre-tokenizer simply splits using the digits in separate tokens
|
| 309 |
+
|
| 310 |
+
Args:
|
| 311 |
+
individual_digits (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
| 312 |
+
If set to True, digits will each be separated as follows::
|
| 313 |
+
|
| 314 |
+
"Call 123 please" -> "Call ", "1", "2", "3", " please"
|
| 315 |
+
|
| 316 |
+
If set to False, digits will grouped as follows::
|
| 317 |
+
|
| 318 |
+
"Call 123 please" -> "Call ", "123", " please"
|
| 319 |
+
"""
|
| 320 |
+
def __init__(self, individual_digits=False):
|
| 321 |
+
pass
|
| 322 |
+
|
| 323 |
+
def __getstate__(self):
|
| 324 |
+
""" """
|
| 325 |
+
pass
|
| 326 |
+
|
| 327 |
+
def __setstate__(self, state):
|
| 328 |
+
""" """
|
| 329 |
+
pass
|
| 330 |
+
|
| 331 |
+
@staticmethod
|
| 332 |
+
def custom(pretok):
|
| 333 |
+
""" """
|
| 334 |
+
pass
|
| 335 |
+
|
| 336 |
+
@property
|
| 337 |
+
def individual_digits(self):
|
| 338 |
+
""" """
|
| 339 |
+
pass
|
| 340 |
+
|
| 341 |
+
@individual_digits.setter
|
| 342 |
+
def individual_digits(self, value):
|
| 343 |
+
""" """
|
| 344 |
+
pass
|
| 345 |
+
|
| 346 |
+
def pre_tokenize(self, pretok):
|
| 347 |
+
"""
|
| 348 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 349 |
+
|
| 350 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 351 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 352 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 353 |
+
the pre-tokenization of a raw string, you can use
|
| 354 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 355 |
+
|
| 356 |
+
Args:
|
| 357 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 358 |
+
The pre-tokenized string on which to apply this
|
| 359 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 360 |
+
"""
|
| 361 |
+
pass
|
| 362 |
+
|
| 363 |
+
def pre_tokenize_str(self, sequence):
|
| 364 |
+
"""
|
| 365 |
+
Pre tokenize the given string
|
| 366 |
+
|
| 367 |
+
This method provides a way to visualize the effect of a
|
| 368 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 369 |
+
alignment, nor does it provide all the capabilities of the
|
| 370 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 371 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 372 |
+
|
| 373 |
+
Args:
|
| 374 |
+
sequence (:obj:`str`):
|
| 375 |
+
A string to pre-tokeize
|
| 376 |
+
|
| 377 |
+
Returns:
|
| 378 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 379 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 380 |
+
"""
|
| 381 |
+
pass
|
| 382 |
+
|
| 383 |
+
class FixedLength(PreTokenizer):
|
| 384 |
+
"""
|
| 385 |
+
This pre-tokenizer splits the text into fixed length chunks as used
|
| 386 |
+
[here](https://www.biorxiv.org/content/10.1101/2023.01.11.523679v1.full)
|
| 387 |
+
|
| 388 |
+
Args:
|
| 389 |
+
length (:obj:`int`, `optional`, defaults to :obj:`5`):
|
| 390 |
+
The length of the chunks to split the text into.
|
| 391 |
+
|
| 392 |
+
Strings are split on the character level rather than the byte level to avoid
|
| 393 |
+
splitting unicode characters consisting of multiple bytes.
|
| 394 |
+
"""
|
| 395 |
+
def __init__(self, length=5):
|
| 396 |
+
pass
|
| 397 |
+
|
| 398 |
+
def __getstate__(self):
|
| 399 |
+
""" """
|
| 400 |
+
pass
|
| 401 |
+
|
| 402 |
+
def __setstate__(self, state):
|
| 403 |
+
""" """
|
| 404 |
+
pass
|
| 405 |
+
|
| 406 |
+
@staticmethod
|
| 407 |
+
def custom(pretok):
|
| 408 |
+
""" """
|
| 409 |
+
pass
|
| 410 |
+
|
| 411 |
+
@property
|
| 412 |
+
def length(self):
|
| 413 |
+
""" """
|
| 414 |
+
pass
|
| 415 |
+
|
| 416 |
+
@length.setter
|
| 417 |
+
def length(self, value):
|
| 418 |
+
""" """
|
| 419 |
+
pass
|
| 420 |
+
|
| 421 |
+
def pre_tokenize(self, pretok):
|
| 422 |
+
"""
|
| 423 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 424 |
+
|
| 425 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 426 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 427 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 428 |
+
the pre-tokenization of a raw string, you can use
|
| 429 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 430 |
+
|
| 431 |
+
Args:
|
| 432 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 433 |
+
The pre-tokenized string on which to apply this
|
| 434 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 435 |
+
"""
|
| 436 |
+
pass
|
| 437 |
+
|
| 438 |
+
def pre_tokenize_str(self, sequence):
|
| 439 |
+
"""
|
| 440 |
+
Pre tokenize the given string
|
| 441 |
+
|
| 442 |
+
This method provides a way to visualize the effect of a
|
| 443 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 444 |
+
alignment, nor does it provide all the capabilities of the
|
| 445 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 446 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 447 |
+
|
| 448 |
+
Args:
|
| 449 |
+
sequence (:obj:`str`):
|
| 450 |
+
A string to pre-tokeize
|
| 451 |
+
|
| 452 |
+
Returns:
|
| 453 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 454 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 455 |
+
"""
|
| 456 |
+
pass
|
| 457 |
+
|
| 458 |
+
class Metaspace(PreTokenizer):
|
| 459 |
+
"""
|
| 460 |
+
Metaspace pre-tokenizer
|
| 461 |
+
|
| 462 |
+
This pre-tokenizer replaces any whitespace by the provided replacement character.
|
| 463 |
+
It then tries to split on these spaces.
|
| 464 |
+
|
| 465 |
+
Args:
|
| 466 |
+
replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
|
| 467 |
+
The replacement character. Must be exactly one character. By default we
|
| 468 |
+
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
|
| 469 |
+
|
| 470 |
+
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
|
| 471 |
+
Whether to add a space to the first word if there isn't already one. This
|
| 472 |
+
lets us treat `hello` exactly like `say hello`.
|
| 473 |
+
Choices: "always", "never", "first". First means the space is only added on the first
|
| 474 |
+
token (relevant when special tokens are used or other pre_tokenizer are used).
|
| 475 |
+
|
| 476 |
+
"""
|
| 477 |
+
def __init__(self, replacement="_", prepend_scheme="always", split=True):
|
| 478 |
+
pass
|
| 479 |
+
|
| 480 |
+
def __getstate__(self):
|
| 481 |
+
""" """
|
| 482 |
+
pass
|
| 483 |
+
|
| 484 |
+
def __setstate__(self, state):
|
| 485 |
+
""" """
|
| 486 |
+
pass
|
| 487 |
+
|
| 488 |
+
@staticmethod
|
| 489 |
+
def custom(pretok):
|
| 490 |
+
""" """
|
| 491 |
+
pass
|
| 492 |
+
|
| 493 |
+
def pre_tokenize(self, pretok):
|
| 494 |
+
"""
|
| 495 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 496 |
+
|
| 497 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 498 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 499 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 500 |
+
the pre-tokenization of a raw string, you can use
|
| 501 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 502 |
+
|
| 503 |
+
Args:
|
| 504 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 505 |
+
The pre-tokenized string on which to apply this
|
| 506 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 507 |
+
"""
|
| 508 |
+
pass
|
| 509 |
+
|
| 510 |
+
def pre_tokenize_str(self, sequence):
|
| 511 |
+
"""
|
| 512 |
+
Pre tokenize the given string
|
| 513 |
+
|
| 514 |
+
This method provides a way to visualize the effect of a
|
| 515 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 516 |
+
alignment, nor does it provide all the capabilities of the
|
| 517 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 518 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 519 |
+
|
| 520 |
+
Args:
|
| 521 |
+
sequence (:obj:`str`):
|
| 522 |
+
A string to pre-tokeize
|
| 523 |
+
|
| 524 |
+
Returns:
|
| 525 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 526 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 527 |
+
"""
|
| 528 |
+
pass
|
| 529 |
+
|
| 530 |
+
@property
|
| 531 |
+
def prepend_scheme(self):
|
| 532 |
+
""" """
|
| 533 |
+
pass
|
| 534 |
+
|
| 535 |
+
@prepend_scheme.setter
|
| 536 |
+
def prepend_scheme(self, value):
|
| 537 |
+
""" """
|
| 538 |
+
pass
|
| 539 |
+
|
| 540 |
+
@property
|
| 541 |
+
def replacement(self):
|
| 542 |
+
""" """
|
| 543 |
+
pass
|
| 544 |
+
|
| 545 |
+
@replacement.setter
|
| 546 |
+
def replacement(self, value):
|
| 547 |
+
""" """
|
| 548 |
+
pass
|
| 549 |
+
|
| 550 |
+
@property
|
| 551 |
+
def split(self):
|
| 552 |
+
""" """
|
| 553 |
+
pass
|
| 554 |
+
|
| 555 |
+
@split.setter
|
| 556 |
+
def split(self, value):
|
| 557 |
+
""" """
|
| 558 |
+
pass
|
| 559 |
+
|
| 560 |
+
class Punctuation(PreTokenizer):
|
| 561 |
+
"""
|
| 562 |
+
This pre-tokenizer simply splits on punctuation as individual characters.
|
| 563 |
+
|
| 564 |
+
Args:
|
| 565 |
+
behavior (:class:`~tokenizers.SplitDelimiterBehavior`):
|
| 566 |
+
The behavior to use when splitting.
|
| 567 |
+
Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next",
|
| 568 |
+
"contiguous"
|
| 569 |
+
"""
|
| 570 |
+
def __init__(self, behavior="isolated"):
|
| 571 |
+
pass
|
| 572 |
+
|
| 573 |
+
def __getstate__(self):
|
| 574 |
+
""" """
|
| 575 |
+
pass
|
| 576 |
+
|
| 577 |
+
def __setstate__(self, state):
|
| 578 |
+
""" """
|
| 579 |
+
pass
|
| 580 |
+
|
| 581 |
+
@property
|
| 582 |
+
def behavior(self):
|
| 583 |
+
""" """
|
| 584 |
+
pass
|
| 585 |
+
|
| 586 |
+
@behavior.setter
|
| 587 |
+
def behavior(self, value):
|
| 588 |
+
""" """
|
| 589 |
+
pass
|
| 590 |
+
|
| 591 |
+
@staticmethod
|
| 592 |
+
def custom(pretok):
|
| 593 |
+
""" """
|
| 594 |
+
pass
|
| 595 |
+
|
| 596 |
+
def pre_tokenize(self, pretok):
|
| 597 |
+
"""
|
| 598 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 599 |
+
|
| 600 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 601 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 602 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 603 |
+
the pre-tokenization of a raw string, you can use
|
| 604 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 605 |
+
|
| 606 |
+
Args:
|
| 607 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 608 |
+
The pre-tokenized string on which to apply this
|
| 609 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 610 |
+
"""
|
| 611 |
+
pass
|
| 612 |
+
|
| 613 |
+
def pre_tokenize_str(self, sequence):
|
| 614 |
+
"""
|
| 615 |
+
Pre tokenize the given string
|
| 616 |
+
|
| 617 |
+
This method provides a way to visualize the effect of a
|
| 618 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 619 |
+
alignment, nor does it provide all the capabilities of the
|
| 620 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 621 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 622 |
+
|
| 623 |
+
Args:
|
| 624 |
+
sequence (:obj:`str`):
|
| 625 |
+
A string to pre-tokeize
|
| 626 |
+
|
| 627 |
+
Returns:
|
| 628 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 629 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 630 |
+
"""
|
| 631 |
+
pass
|
| 632 |
+
|
| 633 |
+
class Sequence(PreTokenizer):
|
| 634 |
+
"""
|
| 635 |
+
This pre-tokenizer composes other pre_tokenizers and applies them in sequence
|
| 636 |
+
"""
|
| 637 |
+
def __init__(self, pretokenizers):
|
| 638 |
+
pass
|
| 639 |
+
|
| 640 |
+
def __getitem__(self, key):
|
| 641 |
+
"""
|
| 642 |
+
Return self[key].
|
| 643 |
+
"""
|
| 644 |
+
pass
|
| 645 |
+
|
| 646 |
+
def __getnewargs__(self):
|
| 647 |
+
""" """
|
| 648 |
+
pass
|
| 649 |
+
|
| 650 |
+
def __getstate__(self):
|
| 651 |
+
""" """
|
| 652 |
+
pass
|
| 653 |
+
|
| 654 |
+
def __setitem__(self, key, value):
|
| 655 |
+
"""
|
| 656 |
+
Set self[key] to value.
|
| 657 |
+
"""
|
| 658 |
+
pass
|
| 659 |
+
|
| 660 |
+
def __setstate__(self, state):
|
| 661 |
+
""" """
|
| 662 |
+
pass
|
| 663 |
+
|
| 664 |
+
@staticmethod
|
| 665 |
+
def custom(pretok):
|
| 666 |
+
""" """
|
| 667 |
+
pass
|
| 668 |
+
|
| 669 |
+
def pre_tokenize(self, pretok):
|
| 670 |
+
"""
|
| 671 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 672 |
+
|
| 673 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 674 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 675 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 676 |
+
the pre-tokenization of a raw string, you can use
|
| 677 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 678 |
+
|
| 679 |
+
Args:
|
| 680 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 681 |
+
The pre-tokenized string on which to apply this
|
| 682 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 683 |
+
"""
|
| 684 |
+
pass
|
| 685 |
+
|
| 686 |
+
def pre_tokenize_str(self, sequence):
|
| 687 |
+
"""
|
| 688 |
+
Pre tokenize the given string
|
| 689 |
+
|
| 690 |
+
This method provides a way to visualize the effect of a
|
| 691 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 692 |
+
alignment, nor does it provide all the capabilities of the
|
| 693 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 694 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 695 |
+
|
| 696 |
+
Args:
|
| 697 |
+
sequence (:obj:`str`):
|
| 698 |
+
A string to pre-tokeize
|
| 699 |
+
|
| 700 |
+
Returns:
|
| 701 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 702 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 703 |
+
"""
|
| 704 |
+
pass
|
| 705 |
+
|
| 706 |
+
class Split(PreTokenizer):
|
| 707 |
+
"""
|
| 708 |
+
Split PreTokenizer
|
| 709 |
+
|
| 710 |
+
This versatile pre-tokenizer splits using the provided pattern and
|
| 711 |
+
according to the provided behavior. The pattern can be inverted by
|
| 712 |
+
making use of the invert flag.
|
| 713 |
+
|
| 714 |
+
Args:
|
| 715 |
+
pattern (:obj:`str` or :class:`~tokenizers.Regex`):
|
| 716 |
+
A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`.
|
| 717 |
+
If you want to use a regex pattern, it has to be wrapped around a `tokenizers.Regex`,
|
| 718 |
+
otherwise we consider is as a string pattern. For example `pattern="|"`
|
| 719 |
+
means you want to split on `|` (imagine a csv file for example), while
|
| 720 |
+
`pattern=tokenizers.Regex("1|2")` means you split on either '1' or '2'.
|
| 721 |
+
behavior (:class:`~tokenizers.SplitDelimiterBehavior`):
|
| 722 |
+
The behavior to use when splitting.
|
| 723 |
+
Choices: "removed", "isolated", "merged_with_previous", "merged_with_next",
|
| 724 |
+
"contiguous"
|
| 725 |
+
|
| 726 |
+
invert (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
| 727 |
+
Whether to invert the pattern.
|
| 728 |
+
"""
|
| 729 |
+
def __init__(self, pattern, behavior, invert=False):
|
| 730 |
+
pass
|
| 731 |
+
|
| 732 |
+
def __getnewargs__(self):
|
| 733 |
+
""" """
|
| 734 |
+
pass
|
| 735 |
+
|
| 736 |
+
def __getstate__(self):
|
| 737 |
+
""" """
|
| 738 |
+
pass
|
| 739 |
+
|
| 740 |
+
def __setstate__(self, state):
|
| 741 |
+
""" """
|
| 742 |
+
pass
|
| 743 |
+
|
| 744 |
+
@property
|
| 745 |
+
def behavior(self):
|
| 746 |
+
""" """
|
| 747 |
+
pass
|
| 748 |
+
|
| 749 |
+
@behavior.setter
|
| 750 |
+
def behavior(self, value):
|
| 751 |
+
""" """
|
| 752 |
+
pass
|
| 753 |
+
|
| 754 |
+
@staticmethod
|
| 755 |
+
def custom(pretok):
|
| 756 |
+
""" """
|
| 757 |
+
pass
|
| 758 |
+
|
| 759 |
+
@property
|
| 760 |
+
def invert(self):
|
| 761 |
+
""" """
|
| 762 |
+
pass
|
| 763 |
+
|
| 764 |
+
@invert.setter
|
| 765 |
+
def invert(self, value):
|
| 766 |
+
""" """
|
| 767 |
+
pass
|
| 768 |
+
|
| 769 |
+
@property
|
| 770 |
+
def pattern(self):
|
| 771 |
+
""" """
|
| 772 |
+
pass
|
| 773 |
+
|
| 774 |
+
@pattern.setter
|
| 775 |
+
def pattern(self, value):
|
| 776 |
+
""" """
|
| 777 |
+
pass
|
| 778 |
+
|
| 779 |
+
def pre_tokenize(self, pretok):
|
| 780 |
+
"""
|
| 781 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 782 |
+
|
| 783 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 784 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 785 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 786 |
+
the pre-tokenization of a raw string, you can use
|
| 787 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 788 |
+
|
| 789 |
+
Args:
|
| 790 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 791 |
+
The pre-tokenized string on which to apply this
|
| 792 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 793 |
+
"""
|
| 794 |
+
pass
|
| 795 |
+
|
| 796 |
+
def pre_tokenize_str(self, sequence):
|
| 797 |
+
"""
|
| 798 |
+
Pre tokenize the given string
|
| 799 |
+
|
| 800 |
+
This method provides a way to visualize the effect of a
|
| 801 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 802 |
+
alignment, nor does it provide all the capabilities of the
|
| 803 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 804 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 805 |
+
|
| 806 |
+
Args:
|
| 807 |
+
sequence (:obj:`str`):
|
| 808 |
+
A string to pre-tokeize
|
| 809 |
+
|
| 810 |
+
Returns:
|
| 811 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 812 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 813 |
+
"""
|
| 814 |
+
pass
|
| 815 |
+
|
| 816 |
+
class UnicodeScripts(PreTokenizer):
|
| 817 |
+
"""
|
| 818 |
+
This pre-tokenizer splits on characters that belong to different language family
|
| 819 |
+
It roughly follows https://github.com/google/sentencepiece/blob/master/data/Scripts.txt
|
| 820 |
+
Actually Hiragana and Katakana are fused with Han, and 0x30FC is Han too.
|
| 821 |
+
This mimicks SentencePiece Unigram implementation.
|
| 822 |
+
"""
|
| 823 |
+
def __init__(self):
|
| 824 |
+
pass
|
| 825 |
+
|
| 826 |
+
def __getstate__(self):
|
| 827 |
+
""" """
|
| 828 |
+
pass
|
| 829 |
+
|
| 830 |
+
def __setstate__(self, state):
|
| 831 |
+
""" """
|
| 832 |
+
pass
|
| 833 |
+
|
| 834 |
+
@staticmethod
|
| 835 |
+
def custom(pretok):
|
| 836 |
+
""" """
|
| 837 |
+
pass
|
| 838 |
+
|
| 839 |
+
def pre_tokenize(self, pretok):
|
| 840 |
+
"""
|
| 841 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 842 |
+
|
| 843 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 844 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 845 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 846 |
+
the pre-tokenization of a raw string, you can use
|
| 847 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 848 |
+
|
| 849 |
+
Args:
|
| 850 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 851 |
+
The pre-tokenized string on which to apply this
|
| 852 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 853 |
+
"""
|
| 854 |
+
pass
|
| 855 |
+
|
| 856 |
+
def pre_tokenize_str(self, sequence):
|
| 857 |
+
"""
|
| 858 |
+
Pre tokenize the given string
|
| 859 |
+
|
| 860 |
+
This method provides a way to visualize the effect of a
|
| 861 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 862 |
+
alignment, nor does it provide all the capabilities of the
|
| 863 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 864 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 865 |
+
|
| 866 |
+
Args:
|
| 867 |
+
sequence (:obj:`str`):
|
| 868 |
+
A string to pre-tokeize
|
| 869 |
+
|
| 870 |
+
Returns:
|
| 871 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 872 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 873 |
+
"""
|
| 874 |
+
pass
|
| 875 |
+
|
| 876 |
+
class Whitespace(PreTokenizer):
|
| 877 |
+
"""
|
| 878 |
+
This pre-tokenizer splits on word boundaries according to the `\w+|[^\w\s]+`
|
| 879 |
+
regex pattern. It splits on word characters or characters that aren't words or
|
| 880 |
+
whitespaces (punctuation such as hyphens, apostrophes, commas, etc.).
|
| 881 |
+
|
| 882 |
+
Example:
|
| 883 |
+
Use the `Whitespace` function as shown below::
|
| 884 |
+
|
| 885 |
+
```python
|
| 886 |
+
from tokenizers.pre_tokenizers import Whitespace
|
| 887 |
+
|
| 888 |
+
pre_tokenizer = Whitespace()
|
| 889 |
+
text = "Hello, world! Let's try the Whitespace pre-tokenizer."
|
| 890 |
+
pre_tokenizer.pre_tokenize_str(text)
|
| 891 |
+
[('Hello', (0, 5)),
|
| 892 |
+
(',', (5, 6)),
|
| 893 |
+
('world', (7, 12)),
|
| 894 |
+
('!', (12, 13)),
|
| 895 |
+
('Let', (14, 17)),
|
| 896 |
+
("'", (17, 18)),
|
| 897 |
+
('s', (18, 19)),
|
| 898 |
+
('try', (20, 23)),
|
| 899 |
+
('the', (24, 27)),
|
| 900 |
+
('Whitespace', (28, 38)),
|
| 901 |
+
('pre', (39, 42)),
|
| 902 |
+
('-', (42, 43)),
|
| 903 |
+
('tokenizer', (43, 52)),
|
| 904 |
+
('.', (52, 53))]
|
| 905 |
+
```
|
| 906 |
+
"""
|
| 907 |
+
def __init__(self):
|
| 908 |
+
pass
|
| 909 |
+
|
| 910 |
+
def __getstate__(self):
|
| 911 |
+
""" """
|
| 912 |
+
pass
|
| 913 |
+
|
| 914 |
+
def __setstate__(self, state):
|
| 915 |
+
""" """
|
| 916 |
+
pass
|
| 917 |
+
|
| 918 |
+
@staticmethod
|
| 919 |
+
def custom(pretok):
|
| 920 |
+
""" """
|
| 921 |
+
pass
|
| 922 |
+
|
| 923 |
+
def pre_tokenize(self, pretok):
|
| 924 |
+
"""
|
| 925 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 926 |
+
|
| 927 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 928 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 929 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 930 |
+
the pre-tokenization of a raw string, you can use
|
| 931 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 932 |
+
|
| 933 |
+
Args:
|
| 934 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 935 |
+
The pre-tokenized string on which to apply this
|
| 936 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 937 |
+
"""
|
| 938 |
+
pass
|
| 939 |
+
|
| 940 |
+
def pre_tokenize_str(self, sequence):
|
| 941 |
+
"""
|
| 942 |
+
Pre tokenize the given string
|
| 943 |
+
|
| 944 |
+
This method provides a way to visualize the effect of a
|
| 945 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 946 |
+
alignment, nor does it provide all the capabilities of the
|
| 947 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 948 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 949 |
+
|
| 950 |
+
Args:
|
| 951 |
+
sequence (:obj:`str`):
|
| 952 |
+
A string to pre-tokeize
|
| 953 |
+
|
| 954 |
+
Returns:
|
| 955 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 956 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 957 |
+
"""
|
| 958 |
+
pass
|
| 959 |
+
|
| 960 |
+
class WhitespaceSplit(PreTokenizer):
|
| 961 |
+
"""
|
| 962 |
+
This pre-tokenizer simply splits on the whitespace. Works like `.split()`
|
| 963 |
+
"""
|
| 964 |
+
def __init__(self):
|
| 965 |
+
pass
|
| 966 |
+
|
| 967 |
+
def __getstate__(self):
|
| 968 |
+
""" """
|
| 969 |
+
pass
|
| 970 |
+
|
| 971 |
+
def __setstate__(self, state):
|
| 972 |
+
""" """
|
| 973 |
+
pass
|
| 974 |
+
|
| 975 |
+
@staticmethod
|
| 976 |
+
def custom(pretok):
|
| 977 |
+
""" """
|
| 978 |
+
pass
|
| 979 |
+
|
| 980 |
+
def pre_tokenize(self, pretok):
|
| 981 |
+
"""
|
| 982 |
+
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
| 983 |
+
|
| 984 |
+
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
| 985 |
+
keep track of the pre-tokenization, and leverage the capabilities of the
|
| 986 |
+
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
| 987 |
+
the pre-tokenization of a raw string, you can use
|
| 988 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
| 989 |
+
|
| 990 |
+
Args:
|
| 991 |
+
pretok (:class:`~tokenizers.PreTokenizedString):
|
| 992 |
+
The pre-tokenized string on which to apply this
|
| 993 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
| 994 |
+
"""
|
| 995 |
+
pass
|
| 996 |
+
|
| 997 |
+
def pre_tokenize_str(self, sequence):
|
| 998 |
+
"""
|
| 999 |
+
Pre tokenize the given string
|
| 1000 |
+
|
| 1001 |
+
This method provides a way to visualize the effect of a
|
| 1002 |
+
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
| 1003 |
+
alignment, nor does it provide all the capabilities of the
|
| 1004 |
+
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
| 1005 |
+
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
| 1006 |
+
|
| 1007 |
+
Args:
|
| 1008 |
+
sequence (:obj:`str`):
|
| 1009 |
+
A string to pre-tokeize
|
| 1010 |
+
|
| 1011 |
+
Returns:
|
| 1012 |
+
:obj:`List[Tuple[str, Offsets]]`:
|
| 1013 |
+
A list of tuple with the pre-tokenized parts and their offsets
|
| 1014 |
+
"""
|
| 1015 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/processors/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
from .. import processors
|
| 3 |
+
|
| 4 |
+
PostProcessor = processors.PostProcessor
|
| 5 |
+
BertProcessing = processors.BertProcessing
|
| 6 |
+
ByteLevel = processors.ByteLevel
|
| 7 |
+
RobertaProcessing = processors.RobertaProcessing
|
| 8 |
+
Sequence = processors.Sequence
|
| 9 |
+
TemplateProcessing = processors.TemplateProcessing
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/processors/__init__.pyi
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class PostProcessor:
|
| 3 |
+
"""
|
| 4 |
+
Base class for all post-processors
|
| 5 |
+
|
| 6 |
+
This class is not supposed to be instantiated directly. Instead, any implementation of
|
| 7 |
+
a PostProcessor will return an instance of this class when instantiated.
|
| 8 |
+
"""
|
| 9 |
+
def __getstate__(self):
|
| 10 |
+
""" """
|
| 11 |
+
pass
|
| 12 |
+
|
| 13 |
+
def __setstate__(self, state):
|
| 14 |
+
""" """
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 18 |
+
"""
|
| 19 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 20 |
+
|
| 21 |
+
Args:
|
| 22 |
+
is_pair (:obj:`bool`):
|
| 23 |
+
Whether the input would be a pair of sequences
|
| 24 |
+
|
| 25 |
+
Returns:
|
| 26 |
+
:obj:`int`: The number of tokens to add
|
| 27 |
+
"""
|
| 28 |
+
pass
|
| 29 |
+
|
| 30 |
+
def process(self, encoding, pair=None, add_special_tokens=True):
|
| 31 |
+
"""
|
| 32 |
+
Post-process the given encodings, generating the final one
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 36 |
+
The encoding for the first sequence
|
| 37 |
+
|
| 38 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 39 |
+
The encoding for the pair sequence
|
| 40 |
+
|
| 41 |
+
add_special_tokens (:obj:`bool`):
|
| 42 |
+
Whether to add the special tokens
|
| 43 |
+
|
| 44 |
+
Return:
|
| 45 |
+
:class:`~tokenizers.Encoding`: The final encoding
|
| 46 |
+
"""
|
| 47 |
+
pass
|
| 48 |
+
|
| 49 |
+
class BertProcessing(PostProcessor):
|
| 50 |
+
"""
|
| 51 |
+
This post-processor takes care of adding the special tokens needed by
|
| 52 |
+
a Bert model:
|
| 53 |
+
|
| 54 |
+
- a SEP token
|
| 55 |
+
- a CLS token
|
| 56 |
+
|
| 57 |
+
Args:
|
| 58 |
+
sep (:obj:`Tuple[str, int]`):
|
| 59 |
+
A tuple with the string representation of the SEP token, and its id
|
| 60 |
+
|
| 61 |
+
cls (:obj:`Tuple[str, int]`):
|
| 62 |
+
A tuple with the string representation of the CLS token, and its id
|
| 63 |
+
"""
|
| 64 |
+
def __init__(self, sep, cls):
|
| 65 |
+
pass
|
| 66 |
+
|
| 67 |
+
def __getnewargs__(self):
|
| 68 |
+
""" """
|
| 69 |
+
pass
|
| 70 |
+
|
| 71 |
+
def __getstate__(self):
|
| 72 |
+
""" """
|
| 73 |
+
pass
|
| 74 |
+
|
| 75 |
+
def __setstate__(self, state):
|
| 76 |
+
""" """
|
| 77 |
+
pass
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def cls(self):
|
| 81 |
+
""" """
|
| 82 |
+
pass
|
| 83 |
+
|
| 84 |
+
@cls.setter
|
| 85 |
+
def cls(self, value):
|
| 86 |
+
""" """
|
| 87 |
+
pass
|
| 88 |
+
|
| 89 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 90 |
+
"""
|
| 91 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
is_pair (:obj:`bool`):
|
| 95 |
+
Whether the input would be a pair of sequences
|
| 96 |
+
|
| 97 |
+
Returns:
|
| 98 |
+
:obj:`int`: The number of tokens to add
|
| 99 |
+
"""
|
| 100 |
+
pass
|
| 101 |
+
|
| 102 |
+
def process(self, encoding, pair=None, add_special_tokens=True):
|
| 103 |
+
"""
|
| 104 |
+
Post-process the given encodings, generating the final one
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 108 |
+
The encoding for the first sequence
|
| 109 |
+
|
| 110 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 111 |
+
The encoding for the pair sequence
|
| 112 |
+
|
| 113 |
+
add_special_tokens (:obj:`bool`):
|
| 114 |
+
Whether to add the special tokens
|
| 115 |
+
|
| 116 |
+
Return:
|
| 117 |
+
:class:`~tokenizers.Encoding`: The final encoding
|
| 118 |
+
"""
|
| 119 |
+
pass
|
| 120 |
+
|
| 121 |
+
@property
|
| 122 |
+
def sep(self):
|
| 123 |
+
""" """
|
| 124 |
+
pass
|
| 125 |
+
|
| 126 |
+
@sep.setter
|
| 127 |
+
def sep(self, value):
|
| 128 |
+
""" """
|
| 129 |
+
pass
|
| 130 |
+
|
| 131 |
+
class ByteLevel(PostProcessor):
|
| 132 |
+
"""
|
| 133 |
+
This post-processor takes care of trimming the offsets.
|
| 134 |
+
|
| 135 |
+
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
| 136 |
+
want the offsets to include these whitespaces, then this PostProcessor must be used.
|
| 137 |
+
|
| 138 |
+
Args:
|
| 139 |
+
trim_offsets (:obj:`bool`):
|
| 140 |
+
Whether to trim the whitespaces from the produced offsets.
|
| 141 |
+
|
| 142 |
+
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 143 |
+
If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments
|
| 144 |
+
the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets`
|
| 145 |
+
is set to :obj:`True`.
|
| 146 |
+
"""
|
| 147 |
+
def __init__(self, add_prefix_space=None, trim_offsets=None, use_regex=None):
|
| 148 |
+
pass
|
| 149 |
+
|
| 150 |
+
def __getstate__(self):
|
| 151 |
+
""" """
|
| 152 |
+
pass
|
| 153 |
+
|
| 154 |
+
def __setstate__(self, state):
|
| 155 |
+
""" """
|
| 156 |
+
pass
|
| 157 |
+
|
| 158 |
+
@property
|
| 159 |
+
def add_prefix_space(self):
|
| 160 |
+
""" """
|
| 161 |
+
pass
|
| 162 |
+
|
| 163 |
+
@add_prefix_space.setter
|
| 164 |
+
def add_prefix_space(self, value):
|
| 165 |
+
""" """
|
| 166 |
+
pass
|
| 167 |
+
|
| 168 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 169 |
+
"""
|
| 170 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 171 |
+
|
| 172 |
+
Args:
|
| 173 |
+
is_pair (:obj:`bool`):
|
| 174 |
+
Whether the input would be a pair of sequences
|
| 175 |
+
|
| 176 |
+
Returns:
|
| 177 |
+
:obj:`int`: The number of tokens to add
|
| 178 |
+
"""
|
| 179 |
+
pass
|
| 180 |
+
|
| 181 |
+
def process(self, encoding, pair=None, add_special_tokens=True):
|
| 182 |
+
"""
|
| 183 |
+
Post-process the given encodings, generating the final one
|
| 184 |
+
|
| 185 |
+
Args:
|
| 186 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 187 |
+
The encoding for the first sequence
|
| 188 |
+
|
| 189 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 190 |
+
The encoding for the pair sequence
|
| 191 |
+
|
| 192 |
+
add_special_tokens (:obj:`bool`):
|
| 193 |
+
Whether to add the special tokens
|
| 194 |
+
|
| 195 |
+
Return:
|
| 196 |
+
:class:`~tokenizers.Encoding`: The final encoding
|
| 197 |
+
"""
|
| 198 |
+
pass
|
| 199 |
+
|
| 200 |
+
@property
|
| 201 |
+
def trim_offsets(self):
|
| 202 |
+
""" """
|
| 203 |
+
pass
|
| 204 |
+
|
| 205 |
+
@trim_offsets.setter
|
| 206 |
+
def trim_offsets(self, value):
|
| 207 |
+
""" """
|
| 208 |
+
pass
|
| 209 |
+
|
| 210 |
+
@property
|
| 211 |
+
def use_regex(self):
|
| 212 |
+
""" """
|
| 213 |
+
pass
|
| 214 |
+
|
| 215 |
+
@use_regex.setter
|
| 216 |
+
def use_regex(self, value):
|
| 217 |
+
""" """
|
| 218 |
+
pass
|
| 219 |
+
|
| 220 |
+
class RobertaProcessing(PostProcessor):
|
| 221 |
+
"""
|
| 222 |
+
This post-processor takes care of adding the special tokens needed by
|
| 223 |
+
a Roberta model:
|
| 224 |
+
|
| 225 |
+
- a SEP token
|
| 226 |
+
- a CLS token
|
| 227 |
+
|
| 228 |
+
It also takes care of trimming the offsets.
|
| 229 |
+
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
| 230 |
+
want the offsets to include these whitespaces, then this PostProcessor should be initialized
|
| 231 |
+
with :obj:`trim_offsets=True`
|
| 232 |
+
|
| 233 |
+
Args:
|
| 234 |
+
sep (:obj:`Tuple[str, int]`):
|
| 235 |
+
A tuple with the string representation of the SEP token, and its id
|
| 236 |
+
|
| 237 |
+
cls (:obj:`Tuple[str, int]`):
|
| 238 |
+
A tuple with the string representation of the CLS token, and its id
|
| 239 |
+
|
| 240 |
+
trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 241 |
+
Whether to trim the whitespaces from the produced offsets.
|
| 242 |
+
|
| 243 |
+
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
| 244 |
+
Whether the add_prefix_space option was enabled during pre-tokenization. This
|
| 245 |
+
is relevant because it defines the way the offsets are trimmed out.
|
| 246 |
+
"""
|
| 247 |
+
def __init__(self, sep, cls, trim_offsets=True, add_prefix_space=True):
|
| 248 |
+
pass
|
| 249 |
+
|
| 250 |
+
def __getnewargs__(self):
|
| 251 |
+
""" """
|
| 252 |
+
pass
|
| 253 |
+
|
| 254 |
+
def __getstate__(self):
|
| 255 |
+
""" """
|
| 256 |
+
pass
|
| 257 |
+
|
| 258 |
+
def __setstate__(self, state):
|
| 259 |
+
""" """
|
| 260 |
+
pass
|
| 261 |
+
|
| 262 |
+
@property
|
| 263 |
+
def add_prefix_space(self):
|
| 264 |
+
""" """
|
| 265 |
+
pass
|
| 266 |
+
|
| 267 |
+
@add_prefix_space.setter
|
| 268 |
+
def add_prefix_space(self, value):
|
| 269 |
+
""" """
|
| 270 |
+
pass
|
| 271 |
+
|
| 272 |
+
@property
|
| 273 |
+
def cls(self):
|
| 274 |
+
""" """
|
| 275 |
+
pass
|
| 276 |
+
|
| 277 |
+
@cls.setter
|
| 278 |
+
def cls(self, value):
|
| 279 |
+
""" """
|
| 280 |
+
pass
|
| 281 |
+
|
| 282 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 283 |
+
"""
|
| 284 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 285 |
+
|
| 286 |
+
Args:
|
| 287 |
+
is_pair (:obj:`bool`):
|
| 288 |
+
Whether the input would be a pair of sequences
|
| 289 |
+
|
| 290 |
+
Returns:
|
| 291 |
+
:obj:`int`: The number of tokens to add
|
| 292 |
+
"""
|
| 293 |
+
pass
|
| 294 |
+
|
| 295 |
+
def process(self, encoding, pair=None, add_special_tokens=True):
|
| 296 |
+
"""
|
| 297 |
+
Post-process the given encodings, generating the final one
|
| 298 |
+
|
| 299 |
+
Args:
|
| 300 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 301 |
+
The encoding for the first sequence
|
| 302 |
+
|
| 303 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 304 |
+
The encoding for the pair sequence
|
| 305 |
+
|
| 306 |
+
add_special_tokens (:obj:`bool`):
|
| 307 |
+
Whether to add the special tokens
|
| 308 |
+
|
| 309 |
+
Return:
|
| 310 |
+
:class:`~tokenizers.Encoding`: The final encoding
|
| 311 |
+
"""
|
| 312 |
+
pass
|
| 313 |
+
|
| 314 |
+
@property
|
| 315 |
+
def sep(self):
|
| 316 |
+
""" """
|
| 317 |
+
pass
|
| 318 |
+
|
| 319 |
+
@sep.setter
|
| 320 |
+
def sep(self, value):
|
| 321 |
+
""" """
|
| 322 |
+
pass
|
| 323 |
+
|
| 324 |
+
@property
|
| 325 |
+
def trim_offsets(self):
|
| 326 |
+
""" """
|
| 327 |
+
pass
|
| 328 |
+
|
| 329 |
+
@trim_offsets.setter
|
| 330 |
+
def trim_offsets(self, value):
|
| 331 |
+
""" """
|
| 332 |
+
pass
|
| 333 |
+
|
| 334 |
+
class Sequence(PostProcessor):
|
| 335 |
+
"""
|
| 336 |
+
Sequence Processor
|
| 337 |
+
|
| 338 |
+
Args:
|
| 339 |
+
processors (:obj:`List[PostProcessor]`)
|
| 340 |
+
The processors that need to be chained
|
| 341 |
+
"""
|
| 342 |
+
def __init__(self, processors):
|
| 343 |
+
pass
|
| 344 |
+
|
| 345 |
+
def __getitem__(self, key):
|
| 346 |
+
"""
|
| 347 |
+
Return self[key].
|
| 348 |
+
"""
|
| 349 |
+
pass
|
| 350 |
+
|
| 351 |
+
def __getnewargs__(self):
|
| 352 |
+
""" """
|
| 353 |
+
pass
|
| 354 |
+
|
| 355 |
+
def __getstate__(self):
|
| 356 |
+
""" """
|
| 357 |
+
pass
|
| 358 |
+
|
| 359 |
+
def __setitem__(self, key, value):
|
| 360 |
+
"""
|
| 361 |
+
Set self[key] to value.
|
| 362 |
+
"""
|
| 363 |
+
pass
|
| 364 |
+
|
| 365 |
+
def __setstate__(self, state):
|
| 366 |
+
""" """
|
| 367 |
+
pass
|
| 368 |
+
|
| 369 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 370 |
+
"""
|
| 371 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 372 |
+
|
| 373 |
+
Args:
|
| 374 |
+
is_pair (:obj:`bool`):
|
| 375 |
+
Whether the input would be a pair of sequences
|
| 376 |
+
|
| 377 |
+
Returns:
|
| 378 |
+
:obj:`int`: The number of tokens to add
|
| 379 |
+
"""
|
| 380 |
+
pass
|
| 381 |
+
|
| 382 |
+
def process(self, encoding, pair=None, add_special_tokens=True):
|
| 383 |
+
"""
|
| 384 |
+
Post-process the given encodings, generating the final one
|
| 385 |
+
|
| 386 |
+
Args:
|
| 387 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 388 |
+
The encoding for the first sequence
|
| 389 |
+
|
| 390 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 391 |
+
The encoding for the pair sequence
|
| 392 |
+
|
| 393 |
+
add_special_tokens (:obj:`bool`):
|
| 394 |
+
Whether to add the special tokens
|
| 395 |
+
|
| 396 |
+
Return:
|
| 397 |
+
:class:`~tokenizers.Encoding`: The final encoding
|
| 398 |
+
"""
|
| 399 |
+
pass
|
| 400 |
+
|
| 401 |
+
class TemplateProcessing(PostProcessor):
|
| 402 |
+
"""
|
| 403 |
+
Provides a way to specify templates in order to add the special tokens to each
|
| 404 |
+
input sequence as relevant.
|
| 405 |
+
|
| 406 |
+
Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to
|
| 407 |
+
delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first
|
| 408 |
+
sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair
|
| 409 |
+
sequences. The final result looks like this:
|
| 410 |
+
|
| 411 |
+
- Single sequence: :obj:`[CLS] Hello there [SEP]`
|
| 412 |
+
- Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]`
|
| 413 |
+
|
| 414 |
+
With the type ids as following::
|
| 415 |
+
|
| 416 |
+
[CLS] ... [SEP] ... [SEP]
|
| 417 |
+
0 0 0 1 1
|
| 418 |
+
|
| 419 |
+
You can achieve such behavior using a TemplateProcessing::
|
| 420 |
+
|
| 421 |
+
TemplateProcessing(
|
| 422 |
+
single="[CLS] $0 [SEP]",
|
| 423 |
+
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
|
| 424 |
+
special_tokens=[("[CLS]", 1), ("[SEP]", 0)],
|
| 425 |
+
)
|
| 426 |
+
|
| 427 |
+
In this example, each input sequence is identified using a ``$`` construct. This identifier
|
| 428 |
+
lets us specify each input sequence, and the type_id to use. When nothing is specified,
|
| 429 |
+
it uses the default values. Here are the different ways to specify it:
|
| 430 |
+
|
| 431 |
+
- Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B``
|
| 432 |
+
- Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ...
|
| 433 |
+
- Specifying both: ``$A:0``, ``$B:1``, ...
|
| 434 |
+
|
| 435 |
+
The same construct is used for special tokens: ``<identifier>(:<type_id>)?``.
|
| 436 |
+
|
| 437 |
+
**Warning**: You must ensure that you are giving the correct tokens/ids as these
|
| 438 |
+
will be added to the Encoding without any further check. If the given ids correspond
|
| 439 |
+
to something totally different in a `Tokenizer` using this `PostProcessor`, it
|
| 440 |
+
might lead to unexpected results.
|
| 441 |
+
|
| 442 |
+
Args:
|
| 443 |
+
single (:obj:`Template`):
|
| 444 |
+
The template used for single sequences
|
| 445 |
+
|
| 446 |
+
pair (:obj:`Template`):
|
| 447 |
+
The template used when both sequences are specified
|
| 448 |
+
|
| 449 |
+
special_tokens (:obj:`Tokens`):
|
| 450 |
+
The list of special tokens used in each sequences
|
| 451 |
+
|
| 452 |
+
Types:
|
| 453 |
+
|
| 454 |
+
Template (:obj:`str` or :obj:`List`):
|
| 455 |
+
- If a :obj:`str` is provided, the whitespace is used as delimiter between tokens
|
| 456 |
+
- If a :obj:`List[str]` is provided, a list of tokens
|
| 457 |
+
|
| 458 |
+
Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`):
|
| 459 |
+
- A :obj:`Tuple` with both a token and its associated ID, in any order
|
| 460 |
+
- A :obj:`dict` with the following keys:
|
| 461 |
+
- "id": :obj:`str` => The special token id, as specified in the Template
|
| 462 |
+
- "ids": :obj:`List[int]` => The associated IDs
|
| 463 |
+
- "tokens": :obj:`List[str]` => The associated tokens
|
| 464 |
+
|
| 465 |
+
The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have
|
| 466 |
+
the same length.
|
| 467 |
+
"""
|
| 468 |
+
def __init__(self, single=None, pair=None, special_tokens=None):
|
| 469 |
+
pass
|
| 470 |
+
|
| 471 |
+
def __getstate__(self):
|
| 472 |
+
""" """
|
| 473 |
+
pass
|
| 474 |
+
|
| 475 |
+
def __setstate__(self, state):
|
| 476 |
+
""" """
|
| 477 |
+
pass
|
| 478 |
+
|
| 479 |
+
def num_special_tokens_to_add(self, is_pair):
|
| 480 |
+
"""
|
| 481 |
+
Return the number of special tokens that would be added for single/pair sentences.
|
| 482 |
+
|
| 483 |
+
Args:
|
| 484 |
+
is_pair (:obj:`bool`):
|
| 485 |
+
Whether the input would be a pair of sequences
|
| 486 |
+
|
| 487 |
+
Returns:
|
| 488 |
+
:obj:`int`: The number of tokens to add
|
| 489 |
+
"""
|
| 490 |
+
pass
|
| 491 |
+
|
| 492 |
+
def process(self, encoding, pair=None, add_special_tokens=True):
|
| 493 |
+
"""
|
| 494 |
+
Post-process the given encodings, generating the final one
|
| 495 |
+
|
| 496 |
+
Args:
|
| 497 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 498 |
+
The encoding for the first sequence
|
| 499 |
+
|
| 500 |
+
pair (:class:`~tokenizers.Encoding`, `optional`):
|
| 501 |
+
The encoding for the pair sequence
|
| 502 |
+
|
| 503 |
+
add_special_tokens (:obj:`bool`):
|
| 504 |
+
Whether to add the special tokens
|
| 505 |
+
|
| 506 |
+
Return:
|
| 507 |
+
:class:`~tokenizers.Encoding`: The final encoding
|
| 508 |
+
"""
|
| 509 |
+
pass
|
| 510 |
+
|
| 511 |
+
@property
|
| 512 |
+
def single(self):
|
| 513 |
+
""" """
|
| 514 |
+
pass
|
| 515 |
+
|
| 516 |
+
@single.setter
|
| 517 |
+
def single(self, value):
|
| 518 |
+
""" """
|
| 519 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tokenizers.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7acb83f5b89136597e0d14b788d82917bf2870df94575bf77e12731c4e49c4df
|
| 3 |
+
size 7395328
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tokenizers.pyi
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
from . import (
|
| 3 |
+
AddedToken as AddedToken,
|
| 4 |
+
Encoding as Encoding,
|
| 5 |
+
NormalizedString as NormalizedString,
|
| 6 |
+
PreTokenizedString as PreTokenizedString,
|
| 7 |
+
Regex as Regex,
|
| 8 |
+
Token as Token,
|
| 9 |
+
Tokenizer as Tokenizer,
|
| 10 |
+
__version__ as __version__,
|
| 11 |
+
decoders as decoders,
|
| 12 |
+
models as models,
|
| 13 |
+
normalizers as normalizers,
|
| 14 |
+
pre_tokenizers as pre_tokenizers,
|
| 15 |
+
processors as processors,
|
| 16 |
+
trainers as trainers,
|
| 17 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tools/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from .visualizer import Annotation, EncodingVisualizer
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tools/visualizer-styles.css
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.tokenized-text {
|
| 2 |
+
width:100%;
|
| 3 |
+
padding:2rem;
|
| 4 |
+
max-height: 400px;
|
| 5 |
+
overflow-y: auto;
|
| 6 |
+
box-sizing:border-box;
|
| 7 |
+
line-height:4rem; /* Lots of space between lines */
|
| 8 |
+
font-family: "Roboto Light", "Ubuntu Light", "Ubuntu", monospace;
|
| 9 |
+
box-shadow: 2px 2px 2px rgba(0,0,0,0.2);
|
| 10 |
+
background-color: rgba(0,0,0,0.01);
|
| 11 |
+
letter-spacing:2px; /* Give some extra separation between chars */
|
| 12 |
+
}
|
| 13 |
+
.non-token{
|
| 14 |
+
/* White space and other things the tokenizer ignores*/
|
| 15 |
+
white-space: pre;
|
| 16 |
+
letter-spacing:4px;
|
| 17 |
+
border-top:1px solid #A0A0A0; /* A gentle border on top and bottom makes tabs more ovious*/
|
| 18 |
+
border-bottom:1px solid #A0A0A0;
|
| 19 |
+
line-height: 1rem;
|
| 20 |
+
height: calc(100% - 2px);
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
.token {
|
| 24 |
+
white-space: pre;
|
| 25 |
+
position:relative;
|
| 26 |
+
color:black;
|
| 27 |
+
letter-spacing:2px;
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
.annotation{
|
| 31 |
+
white-space:nowrap; /* Important - ensures that annotations appears even if the annotated text wraps a line */
|
| 32 |
+
border-radius:4px;
|
| 33 |
+
position:relative;
|
| 34 |
+
width:fit-content;
|
| 35 |
+
}
|
| 36 |
+
.annotation:before {
|
| 37 |
+
/*The before holds the text and the after holds the background*/
|
| 38 |
+
z-index:1000; /* Make sure this is above the background */
|
| 39 |
+
content:attr(data-label); /* The annotations label is on a data attribute */
|
| 40 |
+
color:white;
|
| 41 |
+
position:absolute;
|
| 42 |
+
font-size:1rem;
|
| 43 |
+
text-align:center;
|
| 44 |
+
font-weight:bold;
|
| 45 |
+
|
| 46 |
+
top:1.75rem;
|
| 47 |
+
line-height:0;
|
| 48 |
+
left:0;
|
| 49 |
+
width:100%;
|
| 50 |
+
padding:0.5rem 0;
|
| 51 |
+
/* These make it so an annotation doesn't stretch beyond the annotated text if the label is longer*/
|
| 52 |
+
overflow: hidden;
|
| 53 |
+
white-space: nowrap;
|
| 54 |
+
text-overflow:ellipsis;
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
.annotation:after {
|
| 58 |
+
content:attr(data-label); /* The content defines the width of the annotation*/
|
| 59 |
+
position:absolute;
|
| 60 |
+
font-size:0.75rem;
|
| 61 |
+
text-align:center;
|
| 62 |
+
font-weight:bold;
|
| 63 |
+
text-overflow:ellipsis;
|
| 64 |
+
top:1.75rem;
|
| 65 |
+
line-height:0;
|
| 66 |
+
overflow: hidden;
|
| 67 |
+
white-space: nowrap;
|
| 68 |
+
|
| 69 |
+
left:0;
|
| 70 |
+
width:100%; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/
|
| 71 |
+
|
| 72 |
+
padding:0.5rem 0;
|
| 73 |
+
/* Nast hack below:
|
| 74 |
+
We set the annotations color in code because we don't know the colors at css time.
|
| 75 |
+
But you can't pass a color as a data attribute to get it into the pseudo element (this thing)
|
| 76 |
+
So to get around that, annotations have the color set on them with a style attribute and then we
|
| 77 |
+
can get the color with currentColor.
|
| 78 |
+
Annotations wrap tokens and tokens set the color back to black
|
| 79 |
+
*/
|
| 80 |
+
background-color: currentColor;
|
| 81 |
+
}
|
| 82 |
+
.annotation:hover::after, .annotation:hover::before{
|
| 83 |
+
/* When the user hovers over an annotation expand the label to display in full
|
| 84 |
+
*/
|
| 85 |
+
min-width: fit-content;
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
.annotation:hover{
|
| 89 |
+
/* Emphasize the annotation start end with a border on hover*/
|
| 90 |
+
border-color: currentColor;
|
| 91 |
+
border: 2px solid;
|
| 92 |
+
}
|
| 93 |
+
.special-token:not(:empty){
|
| 94 |
+
/*
|
| 95 |
+
A none empty special token is like UNK (as opposed to CLS which has no representation in the text )
|
| 96 |
+
*/
|
| 97 |
+
position:relative;
|
| 98 |
+
}
|
| 99 |
+
.special-token:empty::before{
|
| 100 |
+
/* Special tokens that don't have text are displayed as pseudo elements so we dont select them with the mouse*/
|
| 101 |
+
content:attr(data-stok);
|
| 102 |
+
background:#202020;
|
| 103 |
+
font-size:0.75rem;
|
| 104 |
+
color:white;
|
| 105 |
+
margin: 0 0.25rem;
|
| 106 |
+
padding: 0.25rem;
|
| 107 |
+
border-radius:4px
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
.special-token:not(:empty):before {
|
| 111 |
+
/* Special tokens that have text (UNK) are displayed above the actual text*/
|
| 112 |
+
content:attr(data-stok);
|
| 113 |
+
position:absolute;
|
| 114 |
+
bottom:1.75rem;
|
| 115 |
+
min-width:100%;
|
| 116 |
+
width:100%;
|
| 117 |
+
height:1rem;
|
| 118 |
+
line-height:1rem;
|
| 119 |
+
font-size:1rem;
|
| 120 |
+
text-align:center;
|
| 121 |
+
color:white;
|
| 122 |
+
font-weight:bold;
|
| 123 |
+
background:#202020;
|
| 124 |
+
border-radius:10%;
|
| 125 |
+
}
|
| 126 |
+
/*
|
| 127 |
+
We want to alternate the color of tokens, but we can't use nth child because tokens might be broken up by annotations
|
| 128 |
+
instead we apply even and odd class at generation time and color them that way
|
| 129 |
+
*/
|
| 130 |
+
.even-token{
|
| 131 |
+
background:#DCDCDC ;
|
| 132 |
+
border: 1px solid #DCDCDC;
|
| 133 |
+
}
|
| 134 |
+
.odd-token{
|
| 135 |
+
background:#A0A0A0;
|
| 136 |
+
border: 1px solid #A0A0A0;
|
| 137 |
+
}
|
| 138 |
+
.even-token.multi-token,.odd-token.multi-token{
|
| 139 |
+
background: repeating-linear-gradient(
|
| 140 |
+
45deg,
|
| 141 |
+
transparent,
|
| 142 |
+
transparent 1px,
|
| 143 |
+
#ccc 1px,
|
| 144 |
+
#ccc 1px
|
| 145 |
+
),
|
| 146 |
+
/* on "bottom" */
|
| 147 |
+
linear-gradient(
|
| 148 |
+
to bottom,
|
| 149 |
+
#FFB6C1,
|
| 150 |
+
#999
|
| 151 |
+
);
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
.multi-token:hover::after {
|
| 155 |
+
content:"This char has more than 1 token"; /* The content defines the width of the annotation*/
|
| 156 |
+
color:white;
|
| 157 |
+
background-color: black;
|
| 158 |
+
position:absolute;
|
| 159 |
+
font-size:0.75rem;
|
| 160 |
+
text-align:center;
|
| 161 |
+
font-weight:bold;
|
| 162 |
+
text-overflow:ellipsis;
|
| 163 |
+
top:1.75rem;
|
| 164 |
+
line-height:0;
|
| 165 |
+
overflow: hidden;
|
| 166 |
+
white-space: nowrap;
|
| 167 |
+
left:0;
|
| 168 |
+
width:fit-content; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/
|
| 169 |
+
padding:0.5rem 0;
|
| 170 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/tools/visualizer.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import itertools
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
from string import Template
|
| 5 |
+
from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
from tokenizers import Encoding, Tokenizer
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
dirname = os.path.dirname(__file__)
|
| 11 |
+
css_filename = os.path.join(dirname, "visualizer-styles.css")
|
| 12 |
+
with open(css_filename) as f:
|
| 13 |
+
css = f.read()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Annotation:
|
| 17 |
+
start: int
|
| 18 |
+
end: int
|
| 19 |
+
label: str
|
| 20 |
+
|
| 21 |
+
def __init__(self, start: int, end: int, label: str):
|
| 22 |
+
self.start = start
|
| 23 |
+
self.end = end
|
| 24 |
+
self.label = label
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
AnnotationList = List[Annotation]
|
| 28 |
+
PartialIntList = List[Optional[int]]
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class CharStateKey(NamedTuple):
|
| 32 |
+
token_ix: Optional[int]
|
| 33 |
+
anno_ix: Optional[int]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class CharState:
|
| 37 |
+
char_ix: Optional[int]
|
| 38 |
+
|
| 39 |
+
def __init__(self, char_ix):
|
| 40 |
+
self.char_ix = char_ix
|
| 41 |
+
|
| 42 |
+
self.anno_ix: Optional[int] = None
|
| 43 |
+
self.tokens: List[int] = []
|
| 44 |
+
|
| 45 |
+
@property
|
| 46 |
+
def token_ix(self):
|
| 47 |
+
return self.tokens[0] if len(self.tokens) > 0 else None
|
| 48 |
+
|
| 49 |
+
@property
|
| 50 |
+
def is_multitoken(self):
|
| 51 |
+
"""
|
| 52 |
+
BPE tokenizers can output more than one token for a char
|
| 53 |
+
"""
|
| 54 |
+
return len(self.tokens) > 1
|
| 55 |
+
|
| 56 |
+
def partition_key(self) -> CharStateKey:
|
| 57 |
+
return CharStateKey(
|
| 58 |
+
token_ix=self.token_ix,
|
| 59 |
+
anno_ix=self.anno_ix,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class Aligned:
|
| 64 |
+
pass
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class EncodingVisualizer:
|
| 68 |
+
"""
|
| 69 |
+
Build an EncodingVisualizer
|
| 70 |
+
|
| 71 |
+
Args:
|
| 72 |
+
|
| 73 |
+
tokenizer (:class:`~tokenizers.Tokenizer`):
|
| 74 |
+
A tokenizer instance
|
| 75 |
+
|
| 76 |
+
default_to_notebook (:obj:`bool`):
|
| 77 |
+
Whether to render html output in a notebook by default
|
| 78 |
+
|
| 79 |
+
annotation_converter (:obj:`Callable`, `optional`):
|
| 80 |
+
An optional (lambda) function that takes an annotation in any format and returns
|
| 81 |
+
an Annotation object
|
| 82 |
+
"""
|
| 83 |
+
|
| 84 |
+
unk_token_regex = re.compile("(.{1}\b)?(unk|oov)(\b.{1})?", flags=re.IGNORECASE)
|
| 85 |
+
|
| 86 |
+
def __init__(
|
| 87 |
+
self,
|
| 88 |
+
tokenizer: Tokenizer,
|
| 89 |
+
default_to_notebook: bool = True,
|
| 90 |
+
annotation_converter: Optional[Callable[[Any], Annotation]] = None,
|
| 91 |
+
):
|
| 92 |
+
if default_to_notebook:
|
| 93 |
+
try:
|
| 94 |
+
from IPython.core.display import HTML, display # type: ignore[attr-defined]
|
| 95 |
+
except ImportError:
|
| 96 |
+
raise Exception(
|
| 97 |
+
"""We couldn't import IPython utils for html display.
|
| 98 |
+
Are you running in a notebook?
|
| 99 |
+
You can also pass `default_to_notebook=False` to get back raw HTML
|
| 100 |
+
"""
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
self.tokenizer = tokenizer
|
| 104 |
+
self.default_to_notebook = default_to_notebook
|
| 105 |
+
self.annotation_coverter = annotation_converter
|
| 106 |
+
pass
|
| 107 |
+
|
| 108 |
+
def __call__(
|
| 109 |
+
self,
|
| 110 |
+
text: str,
|
| 111 |
+
annotations: Optional[List[Any]] = None,
|
| 112 |
+
default_to_notebook: Optional[bool] = None,
|
| 113 |
+
) -> Optional[str]:
|
| 114 |
+
"""
|
| 115 |
+
Build a visualization of the given text
|
| 116 |
+
|
| 117 |
+
Args:
|
| 118 |
+
text (:obj:`str`):
|
| 119 |
+
The text to tokenize
|
| 120 |
+
|
| 121 |
+
annotations (:obj:`List[Annotation]`, `optional`):
|
| 122 |
+
An optional list of annotations of the text. The can either be an annotation class
|
| 123 |
+
or anything else if you instantiated the visualizer with a converter function
|
| 124 |
+
|
| 125 |
+
default_to_notebook (:obj:`bool`, `optional`, defaults to `False`):
|
| 126 |
+
If True, will render the html in a notebook. Otherwise returns an html string.
|
| 127 |
+
|
| 128 |
+
Returns:
|
| 129 |
+
The HTML string if default_to_notebook is False, otherwise (default) returns None and
|
| 130 |
+
renders the HTML in the notebook
|
| 131 |
+
|
| 132 |
+
"""
|
| 133 |
+
final_default_to_notebook = self.default_to_notebook
|
| 134 |
+
if default_to_notebook is not None:
|
| 135 |
+
final_default_to_notebook = default_to_notebook
|
| 136 |
+
if final_default_to_notebook:
|
| 137 |
+
try:
|
| 138 |
+
from IPython.core.display import HTML, display # type: ignore[attr-defined]
|
| 139 |
+
except ImportError:
|
| 140 |
+
raise Exception(
|
| 141 |
+
"""We couldn't import IPython utils for html display.
|
| 142 |
+
Are you running in a notebook?"""
|
| 143 |
+
)
|
| 144 |
+
if annotations is None:
|
| 145 |
+
annotations = []
|
| 146 |
+
if self.annotation_coverter is not None:
|
| 147 |
+
annotations = list(map(self.annotation_coverter, annotations))
|
| 148 |
+
encoding = self.tokenizer.encode(text)
|
| 149 |
+
html = EncodingVisualizer.__make_html(text, encoding, annotations)
|
| 150 |
+
if final_default_to_notebook:
|
| 151 |
+
display(HTML(html))
|
| 152 |
+
else:
|
| 153 |
+
return html
|
| 154 |
+
|
| 155 |
+
@staticmethod
|
| 156 |
+
def calculate_label_colors(annotations: AnnotationList) -> Dict[str, str]:
|
| 157 |
+
"""
|
| 158 |
+
Generates a color palette for all the labels in a given set of annotations
|
| 159 |
+
|
| 160 |
+
Args:
|
| 161 |
+
annotations (:obj:`Annotation`):
|
| 162 |
+
A list of annotations
|
| 163 |
+
|
| 164 |
+
Returns:
|
| 165 |
+
:obj:`dict`: A dictionary mapping labels to colors in HSL format
|
| 166 |
+
"""
|
| 167 |
+
if len(annotations) == 0:
|
| 168 |
+
return {}
|
| 169 |
+
labels = set(map(lambda x: x.label, annotations))
|
| 170 |
+
num_labels = len(labels)
|
| 171 |
+
h_step = int(255 / num_labels)
|
| 172 |
+
if h_step < 20:
|
| 173 |
+
h_step = 20
|
| 174 |
+
s = 32
|
| 175 |
+
l = 64 # noqa: E741
|
| 176 |
+
h = 10
|
| 177 |
+
colors = {}
|
| 178 |
+
|
| 179 |
+
for label in sorted(labels): # sort so we always get the same colors for a given set of labels
|
| 180 |
+
colors[label] = f"hsl({h},{s}%,{l}%)"
|
| 181 |
+
h += h_step
|
| 182 |
+
return colors
|
| 183 |
+
|
| 184 |
+
@staticmethod
|
| 185 |
+
def consecutive_chars_to_html(
|
| 186 |
+
consecutive_chars_list: List[CharState],
|
| 187 |
+
text: str,
|
| 188 |
+
encoding: Encoding,
|
| 189 |
+
):
|
| 190 |
+
"""
|
| 191 |
+
Converts a list of "consecutive chars" into a single HTML element.
|
| 192 |
+
Chars are consecutive if they fall under the same word, token and annotation.
|
| 193 |
+
The CharState class is a named tuple with a "partition_key" method that makes it easy to
|
| 194 |
+
compare if two chars are consecutive.
|
| 195 |
+
|
| 196 |
+
Args:
|
| 197 |
+
consecutive_chars_list (:obj:`List[CharState]`):
|
| 198 |
+
A list of CharStates that have been grouped together
|
| 199 |
+
|
| 200 |
+
text (:obj:`str`):
|
| 201 |
+
The original text being processed
|
| 202 |
+
|
| 203 |
+
encoding (:class:`~tokenizers.Encoding`):
|
| 204 |
+
The encoding returned from the tokenizer
|
| 205 |
+
|
| 206 |
+
Returns:
|
| 207 |
+
:obj:`str`: The HTML span for a set of consecutive chars
|
| 208 |
+
"""
|
| 209 |
+
first = consecutive_chars_list[0]
|
| 210 |
+
if first.char_ix is None:
|
| 211 |
+
# its a special token
|
| 212 |
+
stoken = encoding.tokens[first.token_ix]
|
| 213 |
+
# special tokens are represented as empty spans. We use the data attribute and css
|
| 214 |
+
# magic to display it
|
| 215 |
+
return f'<span class="special-token" data-stoken={stoken}></span>'
|
| 216 |
+
# We're not in a special token so this group has a start and end.
|
| 217 |
+
last = consecutive_chars_list[-1]
|
| 218 |
+
assert first.char_ix is not None
|
| 219 |
+
assert last.char_ix is not None
|
| 220 |
+
start = first.char_ix
|
| 221 |
+
end = last.char_ix + 1
|
| 222 |
+
span_text = text[start:end]
|
| 223 |
+
css_classes = [] # What css classes will we apply on the resulting span
|
| 224 |
+
data_items = {} # What data attributes will we apply on the result span
|
| 225 |
+
if first.token_ix is not None:
|
| 226 |
+
# We can either be in a token or not (e.g. in white space)
|
| 227 |
+
css_classes.append("token")
|
| 228 |
+
if first.is_multitoken:
|
| 229 |
+
css_classes.append("multi-token")
|
| 230 |
+
if first.token_ix % 2:
|
| 231 |
+
# We use this to color alternating tokens.
|
| 232 |
+
# A token might be split by an annotation that ends in the middle of it, so this
|
| 233 |
+
# lets us visually indicate a consecutive token despite its possible splitting in
|
| 234 |
+
# the html markup
|
| 235 |
+
css_classes.append("odd-token")
|
| 236 |
+
else:
|
| 237 |
+
# Like above, but a different color so we can see the tokens alternate
|
| 238 |
+
css_classes.append("even-token")
|
| 239 |
+
if EncodingVisualizer.unk_token_regex.search(encoding.tokens[first.token_ix]) is not None:
|
| 240 |
+
# This is a special token that is in the text. probably UNK
|
| 241 |
+
css_classes.append("special-token")
|
| 242 |
+
# TODO is this the right name for the data attribute ?
|
| 243 |
+
data_items["stok"] = encoding.tokens[first.token_ix]
|
| 244 |
+
else:
|
| 245 |
+
# In this case we are looking at a group/single char that is not tokenized.
|
| 246 |
+
# e.g. white space
|
| 247 |
+
css_classes.append("non-token")
|
| 248 |
+
css = f'''class="{" ".join(css_classes)}"'''
|
| 249 |
+
data = ""
|
| 250 |
+
for key, val in data_items.items():
|
| 251 |
+
data += f' data-{key}="{val}"'
|
| 252 |
+
return f"<span {css} {data} >{span_text}</span>"
|
| 253 |
+
|
| 254 |
+
@staticmethod
|
| 255 |
+
def __make_html(text: str, encoding: Encoding, annotations: AnnotationList) -> str:
|
| 256 |
+
char_states = EncodingVisualizer.__make_char_states(text, encoding, annotations)
|
| 257 |
+
current_consecutive_chars = [char_states[0]]
|
| 258 |
+
prev_anno_ix = char_states[0].anno_ix
|
| 259 |
+
spans = []
|
| 260 |
+
label_colors_dict = EncodingVisualizer.calculate_label_colors(annotations)
|
| 261 |
+
cur_anno_ix = char_states[0].anno_ix
|
| 262 |
+
if cur_anno_ix is not None:
|
| 263 |
+
# If we started in an annotation make a span for it
|
| 264 |
+
anno = annotations[cur_anno_ix]
|
| 265 |
+
label = anno.label
|
| 266 |
+
color = label_colors_dict[label]
|
| 267 |
+
spans.append(f'<span class="annotation" style="color:{color}" data-label="{label}">')
|
| 268 |
+
|
| 269 |
+
for cs in char_states[1:]:
|
| 270 |
+
cur_anno_ix = cs.anno_ix
|
| 271 |
+
if cur_anno_ix != prev_anno_ix:
|
| 272 |
+
# If we've transitioned in or out of an annotation
|
| 273 |
+
spans.append(
|
| 274 |
+
# Create a span from the current consecutive characters
|
| 275 |
+
EncodingVisualizer.consecutive_chars_to_html(
|
| 276 |
+
current_consecutive_chars,
|
| 277 |
+
text=text,
|
| 278 |
+
encoding=encoding,
|
| 279 |
+
)
|
| 280 |
+
)
|
| 281 |
+
current_consecutive_chars = [cs]
|
| 282 |
+
|
| 283 |
+
if prev_anno_ix is not None:
|
| 284 |
+
# if we transitioned out of an annotation close it's span
|
| 285 |
+
spans.append("</span>")
|
| 286 |
+
if cur_anno_ix is not None:
|
| 287 |
+
# If we entered a new annotation make a span for it
|
| 288 |
+
anno = annotations[cur_anno_ix]
|
| 289 |
+
label = anno.label
|
| 290 |
+
color = label_colors_dict[label]
|
| 291 |
+
spans.append(f'<span class="annotation" style="color:{color}" data-label="{label}">')
|
| 292 |
+
prev_anno_ix = cur_anno_ix
|
| 293 |
+
|
| 294 |
+
if cs.partition_key() == current_consecutive_chars[0].partition_key():
|
| 295 |
+
# If the current charchter is in the same "group" as the previous one
|
| 296 |
+
current_consecutive_chars.append(cs)
|
| 297 |
+
else:
|
| 298 |
+
# Otherwise we make a span for the previous group
|
| 299 |
+
spans.append(
|
| 300 |
+
EncodingVisualizer.consecutive_chars_to_html(
|
| 301 |
+
current_consecutive_chars,
|
| 302 |
+
text=text,
|
| 303 |
+
encoding=encoding,
|
| 304 |
+
)
|
| 305 |
+
)
|
| 306 |
+
# An reset the consecutive_char_list to form a new group
|
| 307 |
+
current_consecutive_chars = [cs]
|
| 308 |
+
# All that's left is to fill out the final span
|
| 309 |
+
# TODO I think there is an edge case here where an annotation's span might not close
|
| 310 |
+
spans.append(
|
| 311 |
+
EncodingVisualizer.consecutive_chars_to_html(
|
| 312 |
+
current_consecutive_chars,
|
| 313 |
+
text=text,
|
| 314 |
+
encoding=encoding,
|
| 315 |
+
)
|
| 316 |
+
)
|
| 317 |
+
res = HTMLBody(spans) # Send the list of spans to the body of our html
|
| 318 |
+
return res
|
| 319 |
+
|
| 320 |
+
@staticmethod
|
| 321 |
+
def __make_anno_map(text: str, annotations: AnnotationList) -> PartialIntList:
|
| 322 |
+
"""
|
| 323 |
+
Args:
|
| 324 |
+
text (:obj:`str`):
|
| 325 |
+
The raw text we want to align to
|
| 326 |
+
|
| 327 |
+
annotations (:obj:`AnnotationList`):
|
| 328 |
+
A (possibly empty) list of annotations
|
| 329 |
+
|
| 330 |
+
Returns:
|
| 331 |
+
A list of length len(text) whose entry at index i is None if there is no annotation on
|
| 332 |
+
character i or k, the index of the annotation that covers index i where k is with
|
| 333 |
+
respect to the list of annotations
|
| 334 |
+
"""
|
| 335 |
+
annotation_map = [None] * len(text)
|
| 336 |
+
for anno_ix, a in enumerate(annotations):
|
| 337 |
+
for i in range(a.start, a.end):
|
| 338 |
+
annotation_map[i] = anno_ix
|
| 339 |
+
return annotation_map
|
| 340 |
+
|
| 341 |
+
@staticmethod
|
| 342 |
+
def __make_char_states(text: str, encoding: Encoding, annotations: AnnotationList) -> List[CharState]:
|
| 343 |
+
"""
|
| 344 |
+
For each character in the original text, we emit a tuple representing it's "state":
|
| 345 |
+
|
| 346 |
+
* which token_ix it corresponds to
|
| 347 |
+
* which word_ix it corresponds to
|
| 348 |
+
* which annotation_ix it corresponds to
|
| 349 |
+
|
| 350 |
+
Args:
|
| 351 |
+
text (:obj:`str`):
|
| 352 |
+
The raw text we want to align to
|
| 353 |
+
|
| 354 |
+
annotations (:obj:`List[Annotation]`):
|
| 355 |
+
A (possibly empty) list of annotations
|
| 356 |
+
|
| 357 |
+
encoding: (:class:`~tokenizers.Encoding`):
|
| 358 |
+
The encoding returned from the tokenizer
|
| 359 |
+
|
| 360 |
+
Returns:
|
| 361 |
+
:obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what
|
| 362 |
+
it's state is
|
| 363 |
+
"""
|
| 364 |
+
annotation_map = EncodingVisualizer.__make_anno_map(text, annotations)
|
| 365 |
+
# Todo make this a dataclass or named tuple
|
| 366 |
+
char_states: List[CharState] = [CharState(char_ix) for char_ix in range(len(text))]
|
| 367 |
+
for token_ix, token in enumerate(encoding.tokens):
|
| 368 |
+
offsets = encoding.token_to_chars(token_ix)
|
| 369 |
+
if offsets is not None:
|
| 370 |
+
start, end = offsets
|
| 371 |
+
for i in range(start, end):
|
| 372 |
+
char_states[i].tokens.append(token_ix)
|
| 373 |
+
for char_ix, anno_ix in enumerate(annotation_map):
|
| 374 |
+
char_states[char_ix].anno_ix = anno_ix
|
| 375 |
+
|
| 376 |
+
return char_states
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def HTMLBody(children: List[str], css_styles=css) -> str:
|
| 380 |
+
"""
|
| 381 |
+
Generates the full html with css from a list of html spans
|
| 382 |
+
|
| 383 |
+
Args:
|
| 384 |
+
children (:obj:`List[str]`):
|
| 385 |
+
A list of strings, assumed to be html elements
|
| 386 |
+
|
| 387 |
+
css_styles (:obj:`str`, `optional`):
|
| 388 |
+
Optional alternative implementation of the css
|
| 389 |
+
|
| 390 |
+
Returns:
|
| 391 |
+
:obj:`str`: An HTML string with style markup
|
| 392 |
+
"""
|
| 393 |
+
children_text = "".join(children)
|
| 394 |
+
return f"""
|
| 395 |
+
<html>
|
| 396 |
+
<head>
|
| 397 |
+
<style>
|
| 398 |
+
{css_styles}
|
| 399 |
+
</style>
|
| 400 |
+
</head>
|
| 401 |
+
<body>
|
| 402 |
+
<div class="tokenized-text" dir=auto>
|
| 403 |
+
{children_text}
|
| 404 |
+
</div>
|
| 405 |
+
</body>
|
| 406 |
+
</html>
|
| 407 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/trainers/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
from .. import trainers
|
| 3 |
+
|
| 4 |
+
Trainer = trainers.Trainer
|
| 5 |
+
BpeTrainer = trainers.BpeTrainer
|
| 6 |
+
UnigramTrainer = trainers.UnigramTrainer
|
| 7 |
+
WordLevelTrainer = trainers.WordLevelTrainer
|
| 8 |
+
WordPieceTrainer = trainers.WordPieceTrainer
|
loan-advisor-agent/.venv/Lib/site-packages/tokenizers/trainers/__init__.pyi
ADDED
|
@@ -0,0 +1,462 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Generated content DO NOT EDIT
|
| 2 |
+
class Trainer:
|
| 3 |
+
"""
|
| 4 |
+
Base class for all trainers
|
| 5 |
+
|
| 6 |
+
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
| 7 |
+
Trainer will return an instance of this class when instantiated.
|
| 8 |
+
"""
|
| 9 |
+
def __getstate__(self):
|
| 10 |
+
""" """
|
| 11 |
+
pass
|
| 12 |
+
|
| 13 |
+
def __setstate__(self, state):
|
| 14 |
+
""" """
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
class BpeTrainer(Trainer):
|
| 18 |
+
"""
|
| 19 |
+
Trainer capable of training a BPE model
|
| 20 |
+
|
| 21 |
+
Args:
|
| 22 |
+
vocab_size (:obj:`int`, `optional`):
|
| 23 |
+
The size of the final vocabulary, including all tokens and alphabet.
|
| 24 |
+
|
| 25 |
+
min_frequency (:obj:`int`, `optional`):
|
| 26 |
+
The minimum frequency a pair should have in order to be merged.
|
| 27 |
+
|
| 28 |
+
show_progress (:obj:`bool`, `optional`):
|
| 29 |
+
Whether to show progress bars while training.
|
| 30 |
+
|
| 31 |
+
special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
|
| 32 |
+
A list of special tokens the model should know of.
|
| 33 |
+
|
| 34 |
+
limit_alphabet (:obj:`int`, `optional`):
|
| 35 |
+
The maximum different characters to keep in the alphabet.
|
| 36 |
+
|
| 37 |
+
initial_alphabet (:obj:`List[str]`, `optional`):
|
| 38 |
+
A list of characters to include in the initial alphabet, even
|
| 39 |
+
if not seen in the training dataset.
|
| 40 |
+
If the strings contain more than one character, only the first one
|
| 41 |
+
is kept.
|
| 42 |
+
|
| 43 |
+
continuing_subword_prefix (:obj:`str`, `optional`):
|
| 44 |
+
A prefix to be used for every subword that is not a beginning-of-word.
|
| 45 |
+
|
| 46 |
+
end_of_word_suffix (:obj:`str`, `optional`):
|
| 47 |
+
A suffix to be used for every subword that is a end-of-word.
|
| 48 |
+
|
| 49 |
+
max_token_length (:obj:`int`, `optional`):
|
| 50 |
+
Prevents creating tokens longer than the specified size.
|
| 51 |
+
This can help with reducing polluting your vocabulary with
|
| 52 |
+
highly repetitive tokens like `======` for wikipedia
|
| 53 |
+
|
| 54 |
+
"""
|
| 55 |
+
def __init__(
|
| 56 |
+
self,
|
| 57 |
+
vocab_size=30000,
|
| 58 |
+
min_frequency=0,
|
| 59 |
+
show_progress=True,
|
| 60 |
+
special_tokens=[],
|
| 61 |
+
limit_alphabet=None,
|
| 62 |
+
initial_alphabet=[],
|
| 63 |
+
continuing_subword_prefix=None,
|
| 64 |
+
end_of_word_suffix=None,
|
| 65 |
+
max_token_length=None,
|
| 66 |
+
words={},
|
| 67 |
+
):
|
| 68 |
+
pass
|
| 69 |
+
|
| 70 |
+
def __getstate__(self):
|
| 71 |
+
""" """
|
| 72 |
+
pass
|
| 73 |
+
|
| 74 |
+
def __setstate__(self, state):
|
| 75 |
+
""" """
|
| 76 |
+
pass
|
| 77 |
+
|
| 78 |
+
@property
|
| 79 |
+
def continuing_subword_prefix(self):
|
| 80 |
+
""" """
|
| 81 |
+
pass
|
| 82 |
+
|
| 83 |
+
@continuing_subword_prefix.setter
|
| 84 |
+
def continuing_subword_prefix(self, value):
|
| 85 |
+
""" """
|
| 86 |
+
pass
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def end_of_word_suffix(self):
|
| 90 |
+
""" """
|
| 91 |
+
pass
|
| 92 |
+
|
| 93 |
+
@end_of_word_suffix.setter
|
| 94 |
+
def end_of_word_suffix(self, value):
|
| 95 |
+
""" """
|
| 96 |
+
pass
|
| 97 |
+
|
| 98 |
+
@property
|
| 99 |
+
def initial_alphabet(self):
|
| 100 |
+
""" """
|
| 101 |
+
pass
|
| 102 |
+
|
| 103 |
+
@initial_alphabet.setter
|
| 104 |
+
def initial_alphabet(self, value):
|
| 105 |
+
""" """
|
| 106 |
+
pass
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def limit_alphabet(self):
|
| 110 |
+
""" """
|
| 111 |
+
pass
|
| 112 |
+
|
| 113 |
+
@limit_alphabet.setter
|
| 114 |
+
def limit_alphabet(self, value):
|
| 115 |
+
""" """
|
| 116 |
+
pass
|
| 117 |
+
|
| 118 |
+
@property
|
| 119 |
+
def max_token_length(self):
|
| 120 |
+
""" """
|
| 121 |
+
pass
|
| 122 |
+
|
| 123 |
+
@max_token_length.setter
|
| 124 |
+
def max_token_length(self, value):
|
| 125 |
+
""" """
|
| 126 |
+
pass
|
| 127 |
+
|
| 128 |
+
@property
|
| 129 |
+
def min_frequency(self):
|
| 130 |
+
""" """
|
| 131 |
+
pass
|
| 132 |
+
|
| 133 |
+
@min_frequency.setter
|
| 134 |
+
def min_frequency(self, value):
|
| 135 |
+
""" """
|
| 136 |
+
pass
|
| 137 |
+
|
| 138 |
+
@property
|
| 139 |
+
def show_progress(self):
|
| 140 |
+
""" """
|
| 141 |
+
pass
|
| 142 |
+
|
| 143 |
+
@show_progress.setter
|
| 144 |
+
def show_progress(self, value):
|
| 145 |
+
""" """
|
| 146 |
+
pass
|
| 147 |
+
|
| 148 |
+
@property
|
| 149 |
+
def special_tokens(self):
|
| 150 |
+
""" """
|
| 151 |
+
pass
|
| 152 |
+
|
| 153 |
+
@special_tokens.setter
|
| 154 |
+
def special_tokens(self, value):
|
| 155 |
+
""" """
|
| 156 |
+
pass
|
| 157 |
+
|
| 158 |
+
@property
|
| 159 |
+
def vocab_size(self):
|
| 160 |
+
""" """
|
| 161 |
+
pass
|
| 162 |
+
|
| 163 |
+
@vocab_size.setter
|
| 164 |
+
def vocab_size(self, value):
|
| 165 |
+
""" """
|
| 166 |
+
pass
|
| 167 |
+
|
| 168 |
+
class UnigramTrainer(Trainer):
|
| 169 |
+
"""
|
| 170 |
+
Trainer capable of training a Unigram model
|
| 171 |
+
|
| 172 |
+
Args:
|
| 173 |
+
vocab_size (:obj:`int`):
|
| 174 |
+
The size of the final vocabulary, including all tokens and alphabet.
|
| 175 |
+
|
| 176 |
+
show_progress (:obj:`bool`):
|
| 177 |
+
Whether to show progress bars while training.
|
| 178 |
+
|
| 179 |
+
special_tokens (:obj:`List[Union[str, AddedToken]]`):
|
| 180 |
+
A list of special tokens the model should know of.
|
| 181 |
+
|
| 182 |
+
initial_alphabet (:obj:`List[str]`):
|
| 183 |
+
A list of characters to include in the initial alphabet, even
|
| 184 |
+
if not seen in the training dataset.
|
| 185 |
+
If the strings contain more than one character, only the first one
|
| 186 |
+
is kept.
|
| 187 |
+
|
| 188 |
+
shrinking_factor (:obj:`float`):
|
| 189 |
+
The shrinking factor used at each step of the training to prune the
|
| 190 |
+
vocabulary.
|
| 191 |
+
|
| 192 |
+
unk_token (:obj:`str`):
|
| 193 |
+
The token used for out-of-vocabulary tokens.
|
| 194 |
+
|
| 195 |
+
max_piece_length (:obj:`int`):
|
| 196 |
+
The maximum length of a given token.
|
| 197 |
+
|
| 198 |
+
n_sub_iterations (:obj:`int`):
|
| 199 |
+
The number of iterations of the EM algorithm to perform before
|
| 200 |
+
pruning the vocabulary.
|
| 201 |
+
"""
|
| 202 |
+
def __init__(
|
| 203 |
+
self,
|
| 204 |
+
vocab_size=8000,
|
| 205 |
+
show_progress=True,
|
| 206 |
+
special_tokens=[],
|
| 207 |
+
initial_alphabet=[],
|
| 208 |
+
shrinking_factor=0.75,
|
| 209 |
+
unk_token=None,
|
| 210 |
+
max_piece_length=16,
|
| 211 |
+
n_sub_iterations=2,
|
| 212 |
+
):
|
| 213 |
+
pass
|
| 214 |
+
|
| 215 |
+
def __getstate__(self):
|
| 216 |
+
""" """
|
| 217 |
+
pass
|
| 218 |
+
|
| 219 |
+
def __setstate__(self, state):
|
| 220 |
+
""" """
|
| 221 |
+
pass
|
| 222 |
+
|
| 223 |
+
@property
|
| 224 |
+
def initial_alphabet(self):
|
| 225 |
+
""" """
|
| 226 |
+
pass
|
| 227 |
+
|
| 228 |
+
@initial_alphabet.setter
|
| 229 |
+
def initial_alphabet(self, value):
|
| 230 |
+
""" """
|
| 231 |
+
pass
|
| 232 |
+
|
| 233 |
+
@property
|
| 234 |
+
def show_progress(self):
|
| 235 |
+
""" """
|
| 236 |
+
pass
|
| 237 |
+
|
| 238 |
+
@show_progress.setter
|
| 239 |
+
def show_progress(self, value):
|
| 240 |
+
""" """
|
| 241 |
+
pass
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def special_tokens(self):
|
| 245 |
+
""" """
|
| 246 |
+
pass
|
| 247 |
+
|
| 248 |
+
@special_tokens.setter
|
| 249 |
+
def special_tokens(self, value):
|
| 250 |
+
""" """
|
| 251 |
+
pass
|
| 252 |
+
|
| 253 |
+
@property
|
| 254 |
+
def vocab_size(self):
|
| 255 |
+
""" """
|
| 256 |
+
pass
|
| 257 |
+
|
| 258 |
+
@vocab_size.setter
|
| 259 |
+
def vocab_size(self, value):
|
| 260 |
+
""" """
|
| 261 |
+
pass
|
| 262 |
+
|
| 263 |
+
class WordLevelTrainer(Trainer):
|
| 264 |
+
"""
|
| 265 |
+
Trainer capable of training a WorldLevel model
|
| 266 |
+
|
| 267 |
+
Args:
|
| 268 |
+
vocab_size (:obj:`int`, `optional`):
|
| 269 |
+
The size of the final vocabulary, including all tokens and alphabet.
|
| 270 |
+
|
| 271 |
+
min_frequency (:obj:`int`, `optional`):
|
| 272 |
+
The minimum frequency a pair should have in order to be merged.
|
| 273 |
+
|
| 274 |
+
show_progress (:obj:`bool`, `optional`):
|
| 275 |
+
Whether to show progress bars while training.
|
| 276 |
+
|
| 277 |
+
special_tokens (:obj:`List[Union[str, AddedToken]]`):
|
| 278 |
+
A list of special tokens the model should know of.
|
| 279 |
+
"""
|
| 280 |
+
def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[]):
|
| 281 |
+
pass
|
| 282 |
+
|
| 283 |
+
def __getstate__(self):
|
| 284 |
+
""" """
|
| 285 |
+
pass
|
| 286 |
+
|
| 287 |
+
def __setstate__(self, state):
|
| 288 |
+
""" """
|
| 289 |
+
pass
|
| 290 |
+
|
| 291 |
+
@property
|
| 292 |
+
def min_frequency(self):
|
| 293 |
+
""" """
|
| 294 |
+
pass
|
| 295 |
+
|
| 296 |
+
@min_frequency.setter
|
| 297 |
+
def min_frequency(self, value):
|
| 298 |
+
""" """
|
| 299 |
+
pass
|
| 300 |
+
|
| 301 |
+
@property
|
| 302 |
+
def show_progress(self):
|
| 303 |
+
""" """
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
@show_progress.setter
|
| 307 |
+
def show_progress(self, value):
|
| 308 |
+
""" """
|
| 309 |
+
pass
|
| 310 |
+
|
| 311 |
+
@property
|
| 312 |
+
def special_tokens(self):
|
| 313 |
+
""" """
|
| 314 |
+
pass
|
| 315 |
+
|
| 316 |
+
@special_tokens.setter
|
| 317 |
+
def special_tokens(self, value):
|
| 318 |
+
""" """
|
| 319 |
+
pass
|
| 320 |
+
|
| 321 |
+
@property
|
| 322 |
+
def vocab_size(self):
|
| 323 |
+
""" """
|
| 324 |
+
pass
|
| 325 |
+
|
| 326 |
+
@vocab_size.setter
|
| 327 |
+
def vocab_size(self, value):
|
| 328 |
+
""" """
|
| 329 |
+
pass
|
| 330 |
+
|
| 331 |
+
class WordPieceTrainer(Trainer):
|
| 332 |
+
"""
|
| 333 |
+
Trainer capable of training a WordPiece model
|
| 334 |
+
|
| 335 |
+
Args:
|
| 336 |
+
vocab_size (:obj:`int`, `optional`):
|
| 337 |
+
The size of the final vocabulary, including all tokens and alphabet.
|
| 338 |
+
|
| 339 |
+
min_frequency (:obj:`int`, `optional`):
|
| 340 |
+
The minimum frequency a pair should have in order to be merged.
|
| 341 |
+
|
| 342 |
+
show_progress (:obj:`bool`, `optional`):
|
| 343 |
+
Whether to show progress bars while training.
|
| 344 |
+
|
| 345 |
+
special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
|
| 346 |
+
A list of special tokens the model should know of.
|
| 347 |
+
|
| 348 |
+
limit_alphabet (:obj:`int`, `optional`):
|
| 349 |
+
The maximum different characters to keep in the alphabet.
|
| 350 |
+
|
| 351 |
+
initial_alphabet (:obj:`List[str]`, `optional`):
|
| 352 |
+
A list of characters to include in the initial alphabet, even
|
| 353 |
+
if not seen in the training dataset.
|
| 354 |
+
If the strings contain more than one character, only the first one
|
| 355 |
+
is kept.
|
| 356 |
+
|
| 357 |
+
continuing_subword_prefix (:obj:`str`, `optional`):
|
| 358 |
+
A prefix to be used for every subword that is not a beginning-of-word.
|
| 359 |
+
|
| 360 |
+
end_of_word_suffix (:obj:`str`, `optional`):
|
| 361 |
+
A suffix to be used for every subword that is a end-of-word.
|
| 362 |
+
"""
|
| 363 |
+
def __init__(
|
| 364 |
+
self,
|
| 365 |
+
vocab_size=30000,
|
| 366 |
+
min_frequency=0,
|
| 367 |
+
show_progress=True,
|
| 368 |
+
special_tokens=[],
|
| 369 |
+
limit_alphabet=None,
|
| 370 |
+
initial_alphabet=[],
|
| 371 |
+
continuing_subword_prefix="##",
|
| 372 |
+
end_of_word_suffix=None,
|
| 373 |
+
):
|
| 374 |
+
pass
|
| 375 |
+
|
| 376 |
+
def __getstate__(self):
|
| 377 |
+
""" """
|
| 378 |
+
pass
|
| 379 |
+
|
| 380 |
+
def __setstate__(self, state):
|
| 381 |
+
""" """
|
| 382 |
+
pass
|
| 383 |
+
|
| 384 |
+
@property
|
| 385 |
+
def continuing_subword_prefix(self):
|
| 386 |
+
""" """
|
| 387 |
+
pass
|
| 388 |
+
|
| 389 |
+
@continuing_subword_prefix.setter
|
| 390 |
+
def continuing_subword_prefix(self, value):
|
| 391 |
+
""" """
|
| 392 |
+
pass
|
| 393 |
+
|
| 394 |
+
@property
|
| 395 |
+
def end_of_word_suffix(self):
|
| 396 |
+
""" """
|
| 397 |
+
pass
|
| 398 |
+
|
| 399 |
+
@end_of_word_suffix.setter
|
| 400 |
+
def end_of_word_suffix(self, value):
|
| 401 |
+
""" """
|
| 402 |
+
pass
|
| 403 |
+
|
| 404 |
+
@property
|
| 405 |
+
def initial_alphabet(self):
|
| 406 |
+
""" """
|
| 407 |
+
pass
|
| 408 |
+
|
| 409 |
+
@initial_alphabet.setter
|
| 410 |
+
def initial_alphabet(self, value):
|
| 411 |
+
""" """
|
| 412 |
+
pass
|
| 413 |
+
|
| 414 |
+
@property
|
| 415 |
+
def limit_alphabet(self):
|
| 416 |
+
""" """
|
| 417 |
+
pass
|
| 418 |
+
|
| 419 |
+
@limit_alphabet.setter
|
| 420 |
+
def limit_alphabet(self, value):
|
| 421 |
+
""" """
|
| 422 |
+
pass
|
| 423 |
+
|
| 424 |
+
@property
|
| 425 |
+
def min_frequency(self):
|
| 426 |
+
""" """
|
| 427 |
+
pass
|
| 428 |
+
|
| 429 |
+
@min_frequency.setter
|
| 430 |
+
def min_frequency(self, value):
|
| 431 |
+
""" """
|
| 432 |
+
pass
|
| 433 |
+
|
| 434 |
+
@property
|
| 435 |
+
def show_progress(self):
|
| 436 |
+
""" """
|
| 437 |
+
pass
|
| 438 |
+
|
| 439 |
+
@show_progress.setter
|
| 440 |
+
def show_progress(self, value):
|
| 441 |
+
""" """
|
| 442 |
+
pass
|
| 443 |
+
|
| 444 |
+
@property
|
| 445 |
+
def special_tokens(self):
|
| 446 |
+
""" """
|
| 447 |
+
pass
|
| 448 |
+
|
| 449 |
+
@special_tokens.setter
|
| 450 |
+
def special_tokens(self, value):
|
| 451 |
+
""" """
|
| 452 |
+
pass
|
| 453 |
+
|
| 454 |
+
@property
|
| 455 |
+
def vocab_size(self):
|
| 456 |
+
""" """
|
| 457 |
+
pass
|
| 458 |
+
|
| 459 |
+
@vocab_size.setter
|
| 460 |
+
def vocab_size(self, value):
|
| 461 |
+
""" """
|
| 462 |
+
pass
|